From 13f4abec4c5a0342a534119505ab6421d2221e60 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin <56930273+ianton-ru@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:00:26 +0200 Subject: [PATCH 1/6] Merge branch 'master' into iceberg-puffin-deletion-vectors-read-2 --- .../table-engines/integrations/iceberg.md | 11 +- docs/en/sql-reference/statements/system.md | 4 + .../sql-reference/table-functions/iceberg.md | 17 +- docs/reference/formats/Puffin/Puffin.mdx | 4 + .../formats/Puffin/PuffinMetadata.mdx | 4 + programs/local/LocalServer.cpp | 15 + programs/server/Server.cpp | 15 + src/Access/Common/AccessType.h | 1 + .../AggregateFunctionGroupBitmapData.h | 139 ++++ .../BuzzHouse/Generator/SessionSettings.cpp | 1 + src/Common/CacheBase.h | 50 +- src/Common/CurrentMetrics.cpp | 2 + src/Common/ProfileEvents.cpp | 5 + src/Common/tests/gtest_lru_cache.cpp | 67 ++ src/Core/Defines.h | 4 + src/Core/ServerSettings.cpp | 9 + src/Core/Settings.cpp | 8 + src/Core/SettingsChangesHistory.cpp | 91 +++ src/Interpreters/ClusterFunctionReadTask.cpp | 40 ++ src/Interpreters/Context.cpp | 48 ++ src/Interpreters/Context.h | 6 + src/Interpreters/InterpreterSystemQuery.cpp | 5 + src/Parsers/ASTSystemQuery.cpp | 1 + src/Parsers/ASTSystemQuery.h | 1 + src/Parsers/ParserSystemQuery.cpp | 2 + .../Formats/Impl/Parquet/Reader.cpp | 49 +- src/Processors/Formats/Impl/Parquet/Reader.h | 4 + .../Impl/ParquetV3BlockInputFormat.cpp | 71 +- .../Formats/Impl/ParquetV3BlockInputFormat.h | 11 +- .../Formats/Impl/PuffinBlockInputFormat.cpp | 660 ++---------------- .../Formats/Impl/PuffinBlockInputFormat.h | 13 +- ...gtest_parquet_row_group_global_offsets.cpp | 88 +++ ...est_parquet_v3_need_only_count_buckets.cpp | 192 +++++ .../Common/AvroForIcebergDeserializer.cpp | 41 +- .../DataLakes/DataLakeObjectMetadata.h | 9 + .../DataLakes/DeletionVectorTransform.cpp | 70 ++ .../DataLakes/Iceberg/Constant.h | 2 + .../Iceberg/IcebergDataObjectInfo.cpp | 46 +- .../DataLakes/Iceberg/IcebergDataObjectInfo.h | 14 + .../Iceberg/IcebergDeletionVector.cpp | 273 ++++++++ .../DataLakes/Iceberg/IcebergDeletionVector.h | 38 + .../DataLakes/Iceberg/IcebergIterator.cpp | 138 +++- .../DataLakes/Iceberg/IcebergIterator.h | 4 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 69 +- .../DataLakes/Iceberg/ManifestFile.cpp | 32 + .../DataLakes/Iceberg/ManifestFile.h | 40 +- .../Iceberg/ManifestFileIterator.cpp | 5 + .../DataLakes/Iceberg/ManifestFileIterator.h | 2 + .../DataLakes/Iceberg/Snapshot.h | 14 +- .../tests/gtest_iceberg_count_shortcuts.cpp | 62 ++ .../gtest_iceberg_data_object_info_clone.cpp | 64 ++ ...test_iceberg_deletion_vector_positions.cpp | 90 +++ ...test_iceberg_parquet_row_deletes_guard.cpp | 47 ++ ..._iceberg_position_delete_kind_presence.cpp | 108 +++ .../DataLakes/PuffinDeletionVectorReader.cpp | 306 ++++++++ .../DataLakes/PuffinDeletionVectorReader.h | 32 + .../ObjectStorage/DataLakes/PuffinFile.cpp | 643 +++++++++++++++++ .../ObjectStorage/DataLakes/PuffinFile.h | 55 ++ .../DataLakes/PuffinFilesCache.cpp | 128 ++++ .../DataLakes/PuffinFilesCache.h | 212 ++++++ ...deletion_vector_before_equality_filter.cpp | 68 ++ .../gtest_deletion_vector_need_only_count.cpp | 124 ++++ .../gtest_has_non_empty_excluded_rows.cpp | 28 + .../gtest_puffin_deletion_vector_bounds.cpp | 63 ++ ...est_puffin_deletion_vector_cardinality.cpp | 95 +++ .../gtest_puffin_deletion_vector_envelope.cpp | 124 ++++ ...est_puffin_deletion_vector_footer_bind.cpp | 126 ++++ .../gtest_puffin_dv_referenced_data_file.cpp | 71 ++ .../tests/gtest_puffin_files_cache_clone.cpp | 89 +++ .../tests/gtest_puffin_files_cache_key.cpp | 145 ++++ .../gtest_puffin_files_cache_metrics.cpp | 199 ++++++ .../tests/gtest_puffin_files_cache_weight.cpp | 95 +++ ...gtest_puffin_non_seekable_buffer_limit.cpp | 52 ++ .../ObjectStorage/IObjectIterator.cpp | 8 +- src/Storages/ObjectStorage/IObjectIterator.h | 4 + .../StorageObjectStorageSource.cpp | 85 ++- .../tests/gtest_rendezvous_hashing.cpp | 323 +++++++++ .../test_deletion_vectors.py | 424 +++++++++++ .../01271_show_privileges.reference | 1 + ...117_parser_system_query_variants.reference | 2 + .../04117_parser_system_query_variants.sql | 2 + .../04261_iceberg_deletion_vector.reference | 197 ++++++ .../04261_iceberg_deletion_vector.sh | 16 + ..._iceberg_deletion_vector_complex.reference | 172 +++++ .../04262_iceberg_deletion_vector_complex.sh | 32 + ...04263_iceberg_puffin_files_cache.reference | 19 + .../04263_iceberg_puffin_files_cache.sh | 65 ++ .../04549_puffin_allow_seeks_off.sh | 2 + .../data_minio/dv_puffin_warehouse/README.md | 35 + ...9df-a585-047546b341d9-00001-deletes.puffin | Bin 0 -> 2141 bytes ...ced-935b-d2b8ef79eb76-00001-deletes.puffin | Bin 0 -> 1346 bytes ...d07-9633-95670a571224-00001-deletes.puffin | Bin 0 -> 539 bytes ...3b9-9112-c569f91929b9-00001-deletes.puffin | Bin 0 -> 1337 bytes ...94e-b336-f4d3301291b4-00001-deletes.puffin | Bin 0 -> 1351 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet | Bin 0 -> 779 bytes ...0de-4110-9928-e74531d8b125-0-00005.parquet | Bin 0 -> 1003 bytes ...-de4e-4d07-9633-95670a571224-00001.parquet | Bin 0 -> 969 bytes ...f4f-47e6-ad97-00c53284ff87-0-00001.parquet | Bin 0 -> 1173 bytes ...256-45aa-9ed1-48200445069f-0-00005.parquet | Bin 0 -> 759 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet | Bin 0 -> 766 bytes ...0de-4110-9928-e74531d8b125-0-00001.parquet | Bin 0 -> 1027 bytes ...ebb-4cfc-a3b7-3f2a1034a6dc-0-00001.parquet | Bin 0 -> 1192 bytes ...256-45aa-9ed1-48200445069f-0-00001.parquet | Bin 0 -> 784 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00004.parquet | Bin 0 -> 791 bytes ...0de-4110-9928-e74531d8b125-0-00004.parquet | Bin 0 -> 1043 bytes ...8a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet | Bin 0 -> 1197 bytes ...256-45aa-9ed1-48200445069f-0-00004.parquet | Bin 0 -> 806 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet | Bin 0 -> 800 bytes ...0de-4110-9928-e74531d8b125-0-00002.parquet | Bin 0 -> 1002 bytes ...583-48b6-983a-c2993f261fec-0-00001.parquet | Bin 0 -> 1148 bytes ...256-45aa-9ed1-48200445069f-0-00002.parquet | Bin 0 -> 772 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00003.parquet | Bin 0 -> 800 bytes ...0de-4110-9928-e74531d8b125-0-00003.parquet | Bin 0 -> 1028 bytes ...8dd-4d8a-a31a-6db198e852d4-0-00001.parquet | Bin 0 -> 1176 bytes ...256-45aa-9ed1-48200445069f-0-00003.parquet | Bin 0 -> 766 bytes ...03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro | Bin 0 -> 8221 bytes ...ca5b2b-d3f2-44b4-8494-f0aadd4e0b34-m0.avro | Bin 0 -> 8126 bytes ...1000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro | Bin 0 -> 8145 bytes ...1000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro | Bin 0 -> 8132 bytes ...ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro | Bin 0 -> 8230 bytes ...caec64-2cad-46f5-a966-dd75e3635abc-m0.avro | Bin 0 -> 8135 bytes ...caec64-2cad-46f5-a966-dd75e3635abc-m1.avro | Bin 0 -> 8141 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m0.avro | Bin 0 -> 8142 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m1.avro | Bin 0 -> 8318 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro | Bin 0 -> 8298 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro | Bin 0 -> 8297 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m4.avro | Bin 0 -> 8436 bytes ...19b33f-2be5-4073-820c-82cb1c7d76cb-m0.avro | Bin 0 -> 8179 bytes ...8e784a-6b9c-4894-827d-c56978599ecd-m0.avro | Bin 0 -> 8301 bytes ...1c4572-6006-4649-85b1-47e82e73a396-m0.avro | Bin 0 -> 8157 bytes ...-c21c4572-6006-4649-85b1-47e82e73a396.avro | Bin 0 -> 5015 bytes ...-b68e784a-6b9c-4894-827d-c56978599ecd.avro | Bin 0 -> 5072 bytes ...-23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34.avro | Bin 0 -> 4885 bytes ...-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro | Bin 0 -> 4966 bytes ...-6a19b33f-2be5-4073-820c-82cb1c7d76cb.avro | Bin 0 -> 5124 bytes ...-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro | Bin 0 -> 4908 bytes ...-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro | Bin 0 -> 4800 bytes ...-5b66c606-c296-4fe4-a5f8-d8be592f6b96.avro | Bin 0 -> 5139 bytes ...-41caec64-2cad-46f5-a966-dd75e3635abc.avro | Bin 0 -> 5192 bytes .../metadata/v1.metadata.json | 1 + .../metadata/v10.metadata.json | 1 + .../metadata/v11.metadata.json | 1 + .../metadata/v2.metadata.json | 1 + .../metadata/v3.metadata.json | 1 + .../metadata/v4.metadata.json | 1 + .../metadata/v5.metadata.json | 1 + .../metadata/v6.metadata.json | 1 + .../metadata/v7.metadata.json | 1 + .../metadata/v8.metadata.json | 1 + .../metadata/v9.metadata.json | 1 + .../metadata/version-hint.text | 1 + ...697-43f9-b2ea-9894139e9373-0-00001.parquet | Bin 0 -> 734 bytes ...65e-b4d7-6366f5ebda40-00001-deletes.puffin | Bin 0 -> 532 bytes ...8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro | Bin 0 -> 7802 bytes ...4940c4-6b84-4560-85a3-35e17802d85d-m0.avro | Bin 0 -> 7785 bytes ...-c34940c4-6b84-4560-85a3-35e17802d85d.avro | Bin 0 -> 4789 bytes ...-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro | Bin 0 -> 4870 bytes .../metadata/v1.metadata.json | 1 + .../metadata/v2.metadata.json | 1 + .../metadata/v3.metadata.json | 1 + .../metadata/version-hint.text | 1 + .../data_minio/generate_iceberg_dv_fixture.py | 160 +++++ .../data_puffin/generate_puffin_fixtures.py | 19 +- 163 files changed, 6281 insertions(+), 762 deletions(-) create mode 100644 src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp create mode 100644 src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFile.h create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py create mode 100644 tests/queries/0_stateless/04261_iceberg_deletion_vector.reference create mode 100755 tests/queries/0_stateless/04261_iceberg_deletion_vector.sh create mode 100644 tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference create mode 100755 tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh create mode 100644 tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference create mode 100755 tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-18-0da8af70-7e2e-4ced-935b-d2b8ef79eb76-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-6-74340066-9c57-43b9-9112-c569f91929b9-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-25-ee6fe6d5-2f4f-47e6-ad97-00c53284ff87-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-24-0b9f04c6-aebb-4cfc-a3b7-3f2a1034a6dc-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m4.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/6a19b33f-2be5-4073-820c-82cb1c7d76cb-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-212327459647764229-1-c21c4572-6006-4649-85b1-47e82e73a396.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2245381059480343696-1-23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-399037466005519109-1-6a19b33f-2be5-4073-820c-82cb1c7d76cb.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8541702041949031574-1-5b66c606-c296-4fe4-a5f8-d8be592f6b96.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v1.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v10.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v11.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v2.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v3.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v4.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v5.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v6.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v7.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v8.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v9.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/version-hint.text create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/data/00000-0-e150e3cb-5697-43f9-b2ea-9894139e9373-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/data/00000-2-ee31c25d-1223-465e-b4d7-6366f5ebda40-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/c34940c4-6b84-4560-85a3-35e17802d85d-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v1.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v2.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v3.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/version-hint.text create mode 100644 tests/queries/0_stateless/data_minio/generate_iceberg_dv_fixture.py diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 7a93d3f18c35..f11878792b29 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -131,9 +131,16 @@ ClickHouse supports reading Iceberg tables that use the following deletion metho - [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) - [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) -The following deletion method is **not supported**: -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (introduced in v3) +The following limitations apply to deletion vectors: + +- Only `deletion-vector-v1` Puffin blobs are supported +- Data files must be in Parquet format +- Column-scoped deletion vectors (`fields` in the Puffin blob) are not supported +- Writing deletion vectors is not supported + +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. ### Basic usage {#basic-usage} ```sql diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 69b2b59cb0ae..c00968b24782 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -109,6 +109,10 @@ Clears the per-URL Confluent Schema Registry caches used by the `AvroConfluent` Clears the parquet metadata cache. +## SYSTEM DROP PUFFIN FILES CACHE {#drop-puffin-files-cache} + +Clears the Puffin files cache used for parsed Iceberg puffin file content such as deletion vectors. + ## SYSTEM CLEAR|DROP TEXT INDEX CACHES {#drop-text-index-caches} Clears the text index's header, dictionary and postings caches. diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md index 22f567d590cd..41406616797c 100644 --- a/docs/en/sql-reference/table-functions/iceberg.md +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -120,11 +120,20 @@ ClickHouse supports time travel for Iceberg tables, allowing you to query histor ## Processing of tables with deleted rows {#deleted-rows} -Currently, only Iceberg tables with [position deletes](https://iceberg.apache.org/spec/#position-delete-files) are supported. +ClickHouse supports reading Iceberg tables that use the following deletion methods: -The following deletion methods are **not supported**: -- [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (introduced in v3) +- [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) +- [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) + +The following limitations apply to deletion vectors: + +- Only `deletion-vector-v1` Puffin blobs are supported +- Data files must be in Parquet format +- Column-scoped deletion vectors (`fields` in the Puffin blob) are not supported +- Writing deletion vectors is not supported + +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. ### Basic usage {#basic-usage} diff --git a/docs/reference/formats/Puffin/Puffin.mdx b/docs/reference/formats/Puffin/Puffin.mdx index 96f321ad4d0a..730560d857f3 100644 --- a/docs/reference/formats/Puffin/Puffin.mdx +++ b/docs/reference/formats/Puffin/Puffin.mdx @@ -1,6 +1,10 @@ --- description: 'Documentation for the Puffin format' +input_format: true +output_format: false keywords: ['Puffin'] +sidebar_label: 'Puffin' +sidebar_position: 1 slug: /interfaces/formats/Puffin title: 'Puffin' doc_type: 'reference' diff --git a/docs/reference/formats/Puffin/PuffinMetadata.mdx b/docs/reference/formats/Puffin/PuffinMetadata.mdx index cace329a9705..01ffacdc0198 100644 --- a/docs/reference/formats/Puffin/PuffinMetadata.mdx +++ b/docs/reference/formats/Puffin/PuffinMetadata.mdx @@ -1,6 +1,10 @@ --- description: 'Documentation for the PuffinMetadata format' +input_format: true +output_format: false keywords: ['PuffinMetadata'] +sidebar_label: 'PuffinMetadata' +sidebar_position: 2 slug: /interfaces/formats/PuffinMetadata title: 'PuffinMetadata' doc_type: 'reference' diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b5b2c9ecb065..ad0789866929 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -172,6 +172,10 @@ namespace ServerSetting extern const ServerSettingsUInt64 parquet_metadata_cache_size; extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries; extern const ServerSettingsDouble parquet_metadata_cache_size_ratio; + extern const ServerSettingsString puffin_files_cache_policy; + extern const ServerSettingsUInt64 puffin_files_cache_size; + extern const ServerSettingsUInt64 puffin_files_cache_max_entries; + extern const ServerSettingsDouble puffin_files_cache_size_ratio; extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size; extern const ServerSettingsUInt64 max_io_thread_pool_free_size; extern const ServerSettingsUInt64 max_io_thread_pool_size; @@ -1540,6 +1544,17 @@ void LocalServer::processConfig() global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio); #endif + String puffin_files_cache_policy = server_settings[ServerSetting::puffin_files_cache_policy]; + size_t puffin_files_cache_size = server_settings[ServerSetting::puffin_files_cache_size]; + size_t puffin_files_cache_max_entries = server_settings[ServerSetting::puffin_files_cache_max_entries]; + double puffin_files_cache_size_ratio = server_settings[ServerSetting::puffin_files_cache_size_ratio]; + if (puffin_files_cache_size > max_cache_size) + { + puffin_files_cache_size = max_cache_size; + LOG_INFO(log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(puffin_files_cache_size)); + } + global_context->setPuffinFilesCache(puffin_files_cache_policy, puffin_files_cache_size, puffin_files_cache_max_entries, puffin_files_cache_size_ratio); + Names allowed_disks_table_engines; splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value); global_context->setAllowedDisksForTableEngines(std::unordered_set(allowed_disks_table_engines.begin(), allowed_disks_table_engines.end())); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9260a3c9d2dc..b906c352312b 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -288,6 +288,10 @@ namespace ServerSetting extern const ServerSettingsUInt64 parquet_metadata_cache_size; extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries; extern const ServerSettingsDouble parquet_metadata_cache_size_ratio; + extern const ServerSettingsString puffin_files_cache_policy; + extern const ServerSettingsUInt64 puffin_files_cache_size; + extern const ServerSettingsUInt64 puffin_files_cache_max_entries; + extern const ServerSettingsDouble puffin_files_cache_size_ratio; extern const ServerSettingsUInt64 io_thread_pool_queue_size; extern const ServerSettingsBool jemalloc_enable_global_profiler; extern const ServerSettingsBool jemalloc_collect_global_profile_samples_in_trace_log; @@ -2277,6 +2281,16 @@ try } global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio); #endif + String puffin_files_cache_policy = server_settings[ServerSetting::puffin_files_cache_policy]; + size_t puffin_files_cache_size = server_settings[ServerSetting::puffin_files_cache_size]; + size_t puffin_files_cache_max_entries = server_settings[ServerSetting::puffin_files_cache_max_entries]; + double puffin_files_cache_size_ratio = server_settings[ServerSetting::puffin_files_cache_size_ratio]; + if (puffin_files_cache_size > max_cache_size) + { + puffin_files_cache_size = max_cache_size; + LOG_INFO(log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(puffin_files_cache_size)); + } + global_context->setPuffinFilesCache(puffin_files_cache_policy, puffin_files_cache_size, puffin_files_cache_max_entries, puffin_files_cache_size_ratio); Names allowed_disks_table_engines; splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value); @@ -2706,6 +2720,7 @@ try #if USE_PARQUET global_context->updateParquetMetadataCacheConfiguration(config(), max_cache_size_in_bytes); #endif + global_context->updatePuffinFilesCacheConfiguration(config(), max_cache_size_in_bytes); } #if USE_SSL diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 38766a2ed170..6f6fc8f50eef 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -318,6 +318,7 @@ enum class AccessType : uint8_t M(SYSTEM_DROP_ICEBERG_METADATA_CACHE, "SYSTEM CLEAR ICEBERG_METADATA_CACHE, SYSTEM DROP ICEBERG_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_AVRO_SCHEMA_CACHE, "SYSTEM CLEAR AVRO SCHEMA CACHE, SYSTEM DROP AVRO SCHEMA CACHE, DROP AVRO SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ + M(SYSTEM_DROP_PUFFIN_FILES_CACHE, "SYSTEM DROP PUFFIN FILES CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PRIMARY_INDEX_CACHE, "SYSTEM CLEAR PRIMARY INDEX CACHE, SYSTEM DROP PRIMARY INDEX, DROP PRIMARY INDEX CACHE, DROP PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM CLEAR UNCOMPRESSED CACHE, SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index fde3743850ba..7fa193574639 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -34,6 +34,72 @@ enum BitmapKind Bitmap = 1 }; +/// Approximate heap footprint of a 32-bit CRoaring bitmap (index capacity + container capacities). +/// Unlike `Roaring::getSizeInBytes()`, array/run containers use allocated capacity, not cardinality. +inline UInt64 estimateRoaring32AllocatedBytes(const roaring::Roaring & bitmap) +{ + using namespace roaring::internal; + const roaring::roaring_array_t * ra = &bitmap.roaring.high_low_container; + UInt64 bytes = sizeof(roaring::Roaring); + if (ra->allocation_size > 0) + { + bytes += static_cast(ra->allocation_size) + * (sizeof(uint16_t) + sizeof(uint8_t) + sizeof(container_t *)); + } + for (int32_t i = 0; i < ra->size; ++i) + { + uint8_t typecode = ra->typecodes[i]; + const container_t * c = container_unwrap_shared(ra->containers[i], &typecode); + switch (typecode) + { + case ARRAY_CONTAINER_TYPE: + { + const array_container_t * ac = const_CAST_array(c); + bytes += sizeof(array_container_t) + static_cast(ac->capacity) * sizeof(uint16_t); + break; + } + case BITSET_CONTAINER_TYPE: + { + bytes += sizeof(bitset_container_t) + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: + { + const run_container_t * rc = const_CAST_run(c); + bytes += sizeof(run_container_t) + static_cast(rc->capacity) * sizeof(rle16_t); + break; + } + default: + break; + } + } + return bytes; +} + +/// Count distinct high-32 keys in a Roaring64Map. Fast-path when min/max share one key. +inline UInt64 countRoaring64MapHighKeys(const roaring::Roaring64Map & bitmap) +{ + if (bitmap.isEmpty()) + return 0; + + const UInt64 min_value = bitmap.minimum(); + const UInt64 max_value = bitmap.maximum(); + if ((min_value >> 32) == (max_value >> 32)) + return 1; + + UInt64 keys = 0; + UInt64 prev_high = ~UInt64{0}; + for (auto it = bitmap.begin(); it != bitmap.end(); ++it) + { + const UInt64 high = static_cast(*it) >> 32; + if (high != prev_high) + { + ++keys; + prev_high = high; + } + } + return keys; +} /** * For a small number of values - an array of fixed size "on the stack". @@ -93,6 +159,34 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable return roaring_bitmap->cardinality(); } + UInt64 getAllocatedBytes() const + { + if (isSmall()) + return sizeof(small); + + /// Prefer a heap estimate over `getSizeInBytes()` (serialization size). Include a small + /// allowance for the shared_ptr control block that owns `roaring_bitmap`. + constexpr UInt64 SHARED_PTR_CONTROL_BLOCK = 32; + + if constexpr (sizeof(T) < 8) + { + return estimateRoaring32AllocatedBytes(*roaring_bitmap) + SHARED_PTR_CONTROL_BLOCK; + } + else + { + /// Roaring64Map keeps roarings private; approximate as native serialization size plus + /// per-high-key map/Roaring/container overhead (serialization undercounts capacity and + /// std::map nodes — important for sparse high keys). + constexpr UInt64 PER_HIGH_KEY_OVERHEAD = + 4 * sizeof(void *) + sizeof(UInt32) + sizeof(roaring::Roaring) + 64; + + const UInt64 serialized = roaring_bitmap->getSizeInBytes(/*portable=*/false); + const UInt64 high_keys = countRoaring64MapHighKeys(*roaring_bitmap); + return serialized + high_keys * PER_HIGH_KEY_OVERHEAD + sizeof(roaring::Roaring64Map) + + SHARED_PTR_CONTROL_BLOCK; + } + } + void merge(const RoaringBitmapWithSmallSet & r1) { if (r1.isLarge()) @@ -535,6 +629,51 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable return count; } + /** + * Count set bits in `[range_start, range_end)` without allocating a result bitmap. + * Used by need-only-count DV filtering to avoid an O(N) dense Filter over file rows. + * Implemented via roaring `rank` so repeated per-row-group queries stay O(containers), + * not O(row_groups × cardinality). + */ + UInt64 rb_range_cardinality(UInt64 range_start, UInt64 range_end) const /// NOLINT + { + if (range_start >= range_end) + return 0; + + if (isSmall()) + { + UInt64 count = 0; + for (const auto & x : small) + { + const UInt64 val = static_cast(x.getValue()); + if (val >= range_start && val < range_end) + ++count; + } + return count; + } + + /// |bitmap ∩ [start, end)| = rank(end - 1) - rank(start - 1). Same formula as DeleteBitmap. + if constexpr (sizeof(T) < 8) + { + constexpr UInt64 max_row = std::numeric_limits::max(); + if (range_start > max_row) + return 0; + const UInt64 hi_inclusive = std::min(range_end - 1, max_row); + if (hi_inclusive < range_start) + return 0; + const UInt64 upper = roaring_bitmap->rank(static_cast(hi_inclusive)); + const UInt64 lower = (range_start == 0) ? 0 : roaring_bitmap->rank(static_cast(range_start - 1)); + return upper - lower; + } + else + { + const UInt64 hi_inclusive = range_end - 1; + const UInt64 upper = roaring_bitmap->rank(hi_inclusive); + const UInt64 lower = (range_start == 0) ? 0 : roaring_bitmap->rank(range_start - 1); + return upper - lower; + } + } + /** * Return new set of the smallest `limit` values in set which is no less than `range_start`. * It's used in subset and currently only support UInt32 diff --git a/src/Client/BuzzHouse/Generator/SessionSettings.cpp b/src/Client/BuzzHouse/Generator/SessionSettings.cpp index b4d56f78d57d..cdc597a90780 100644 --- a/src/Client/BuzzHouse/Generator/SessionSettings.cpp +++ b/src/Client/BuzzHouse/Generator/SessionSettings.cpp @@ -1645,6 +1645,7 @@ static std::unordered_map serverSettings2 = { {"use_page_cache_for_local_disks", trueOrFalseSetting}, {"use_page_cache_for_object_storage", trueOrFalseSetting}, {"use_parquet_metadata_cache", trueOrFalseSetting}, + {"use_puffin_files_cache", trueOrFalseSetting}, {"use_query_cache", trueOrFalseSetting}, {"use_roaring_bitmap_iceberg_positional_deletes", trueOrFalseSetting}, {"use_skip_indexes_if_final_exact_mode", CHSetting(trueOrFalse, {"0", "1"}, true)}, diff --git a/src/Common/CacheBase.h b/src/Common/CacheBase.h index 3ca63aa79488..704aab2edc5e 100644 --- a/src/Common/CacheBase.h +++ b/src/Common/CacheBase.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,19 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +/// Outcome of `CacheBase::getOrSetWithOutcome`, decided under the cache lock together with the +/// returned value so callers can classify hits/misses without a follow-up `contains()` race. +enum class CacheGetOrSetOutcome : uint8_t +{ + /// Value was resident when returned (this caller did not run `load_func`). + Hit, + /// This caller ran `load_func` and inserted the value into the cache. + MissInserted, + /// Value was produced (by this caller or a stampede peer) but is not resident — e.g. a concurrent + /// `clear()` discarded the insert token. The returned `MappedPtr` is still valid for the caller. + MissNotResident, +}; + /// Thread-safe cache that evicts entries using special cache policy /// (default policy evicts entries which are not used for a long time). /// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size) @@ -156,9 +170,9 @@ class CacheBase /// Exceptions occurring in load_func will be propagated to the caller. Another thread from the /// set of concurrent threads will then try to call its load_func etc. /// - /// Returns std::pair of the cached value and a bool indicating whether the value was produced during this call. + /// Returns the value together with an outcome that is atomic with residency (see CacheGetOrSetOutcome). template - std::pair getOrSet(const Key & key, LoadFunc && load_func) + std::pair getOrSetWithOutcome(const Key & key, LoadFunc && load_func) { InsertTokenHolder token_holder; { @@ -167,7 +181,7 @@ class CacheBase if (val) { ++hits; - return std::make_pair(val, false); + return std::make_pair(val, CacheGetOrSetOutcome::Hit); } auto & token = insert_tokens[key]; @@ -186,8 +200,18 @@ class CacheBase if (token->value) { /// Another thread already produced the value while we waited for token->mutex. - ++hits; - return std::make_pair(token->value, false); + /// If a concurrent clear() discarded that insert, the value is not resident — + /// count a miss (same class of outcome as the producer when insertion is skipped). + { + std::lock_guard cache_lock(mutex); + if (auto cached = cache_policy->get(key)) + { + ++hits; + return std::make_pair(std::move(cached), CacheGetOrSetOutcome::Hit); + } + } + ++misses; + return std::make_pair(token->value, CacheGetOrSetOutcome::MissNotResident); } ++misses; @@ -197,18 +221,28 @@ class CacheBase /// Insert the new value only if the token is still in present in insert_tokens. /// (The token may be absent because of a concurrent clear() call). - bool result = false; auto token_it = insert_tokens.find(key); if (token_it != insert_tokens.end() && token_it->second.get() == token) { cache_policy->set(key, token->value); - result = true; + if (!token->cleaned_up) + token_holder.cleanup(token_lock, cache_lock); + return std::make_pair(token->value, CacheGetOrSetOutcome::MissInserted); } if (!token->cleaned_up) token_holder.cleanup(token_lock, cache_lock); - return std::make_pair(token->value, result); + return std::make_pair(token->value, CacheGetOrSetOutcome::MissNotResident); + } + + /// Same as getOrSetWithOutcome, but the bool is true only when this call inserted the value + /// (`CacheGetOrSetOutcome::MissInserted`). Prefer getOrSetWithOutcome when classifying hits/misses. + template + std::pair getOrSet(const Key & key, LoadFunc && load_func) + { + auto [value, outcome] = getOrSetWithOutcome(key, std::forward(load_func)); + return std::make_pair(std::move(value), outcome == CacheGetOrSetOutcome::MissInserted); } void getStats(size_t & out_hits, size_t & out_misses) const diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index cdb399e249ce..1d261ce3d14f 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -337,6 +337,8 @@ M(IcebergMetadataFilesCacheFiles, "Number of cached files in the Iceberg metadata cache") \ M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \ M(ParquetMetadataCacheFiles, "Number of cached files in the Parquet metadata cache") \ + M(PuffinFilesCacheBytes, "Size of the Puffin files cache in bytes") \ + M(PuffinFilesCacheFiles, "Number of cached entries in the Puffin files cache") \ M(AvroSchemaCacheBytes, "Size of the Avro schema cache in bytes") \ M(AvroSchemaCacheCells, "Number of cached Avro schemas, including both registered and fetched schemas.") \ M(AvroSchemaRegistryCacheBytes, "Size of the Avro schema registry cache in bytes") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 7f35ef869e92..cb3b2cdc5e83 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -110,6 +110,11 @@ M(IcebergMetadataReturnedObjectInfos, "Total number of returned object infos from iceberg iterator.", ValueType::Number) \ M(IcebergMinMaxNonPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ M(IcebergMinMaxPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ + M(PuffinFilesRead, "Number of Puffin files read (footer or deletion vector blob).", ValueType::Number) \ + M(PuffinFileReadMicroseconds, "Total time spent reading Puffin files.", ValueType::Microseconds) \ + M(PuffinFilesCacheHits, "Number of times parsed Puffin file content has been found in the cache.", ValueType::Number) \ + M(PuffinFilesCacheMisses, "Number of times parsed Puffin file content has not been found in the cache and had to be read from disk.", ValueType::Number) \ + M(PuffinFilesCacheWeightLost, "Approximate number of bytes evicted from the Puffin files cache.", ValueType::Number) \ M(VectorSimilarityIndexCacheHits, "Number of times an index granule has been found in the vector index cache.", ValueType::Number) \ M(VectorSimilarityIndexCacheMisses, "Number of times an index granule has not been found in the vector index cache and had to be read from disk.", ValueType::Number) \ M(VectorSimilarityIndexCacheWeightLost, "Approximate number of bytes evicted from the vector index cache.", ValueType::Number) \ diff --git a/src/Common/tests/gtest_lru_cache.cpp b/src/Common/tests/gtest_lru_cache.cpp index ed9c3c34c45f..ff6ddf4df52c 100644 --- a/src/Common/tests/gtest_lru_cache.cpp +++ b/src/Common/tests/gtest_lru_cache.cpp @@ -105,6 +105,73 @@ TEST(LRUCache, getOrSet) ASSERT_TRUE(*value == 10); } +TEST(LRUCache, getOrSetWithOutcomeHitAndMissInserted) +{ + using SimpleCacheBase = DB::CacheBase; + SimpleCacheBase cache("LRU", CurrentMetrics::end(), CurrentMetrics::end(), /*max_size_in_bytes*/ 10, /*max_count*/ 10, /*size_ratio*/ 0.5); + + size_t loads = 0; + auto load = [&]() + { + ++loads; + return std::make_shared(42); + }; + + { + auto [value, outcome] = cache.getOrSetWithOutcome(1, load); + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 42); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::MissInserted); + EXPECT_EQ(loads, 1u); + } + { + auto [value, outcome] = cache.getOrSetWithOutcome(1, load); + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 42); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::Hit); + EXPECT_EQ(loads, 1u); + } +} + +TEST(LRUCache, getOrSetWithOutcomeClearDuringLoadIsMissNotResident) +{ + using SimpleCacheBase = DB::CacheBase; + SimpleCacheBase cache("LRU", CurrentMetrics::end(), CurrentMetrics::end(), /*max_size_in_bytes*/ 10, /*max_count*/ 10, /*size_ratio*/ 0.5); + + auto [value, outcome] = cache.getOrSetWithOutcome( + 1, + [&]() + { + cache.clear(); + return std::make_shared(7); + }); + + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 7); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::MissNotResident); + EXPECT_FALSE(cache.contains(1)); +} + +TEST(LRUCache, getOrSetWithOutcomeHitRemainsHitIfClearedAfterReturn) +{ + using SimpleCacheBase = DB::CacheBase; + SimpleCacheBase cache("LRU", CurrentMetrics::end(), CurrentMetrics::end(), /*max_size_in_bytes*/ 10, /*max_count*/ 10, /*size_ratio*/ 0.5); + + cache.getOrSetWithOutcome(1, []() { return std::make_shared(1); }); + + auto [value, outcome] = cache.getOrSetWithOutcome(1, []() { return std::make_shared(2); }); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::Hit); + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 1); + + /// A follow-up contains() after clear would return false; callers must use `outcome`, not a + /// second residency check, when classifying hits (PuffinFilesCache used to race here). + cache.clear(); + EXPECT_FALSE(cache.contains(1)); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::Hit); + EXPECT_EQ(*value, 1); +} + TEST(LRUCache, noOnRemoveEntryCallback) { diff --git a/src/Core/Defines.h b/src/Core/Defines.h index c99199f76446..b351507566e8 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -124,6 +124,10 @@ static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_POLICY = "SLRU"; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE = 512_MiB; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO = 0.5; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES = 5000; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_POLICY = "SLRU"; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE = 512_MiB; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_SIZE_RATIO = 0.5; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES = 5000; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_POLICY = "SLRU"; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_MAX_SIZE = 100_MiB; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_SIZE_RATIO = 0.5l; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 1f6eb7e54653..871c83ea7e55 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -24,6 +24,7 @@ #if USE_PARQUET # include #endif +#include #include #include #include @@ -560,6 +561,10 @@ namespace DECLARE(UInt64, parquet_metadata_cache_size, DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE, "Maximum size of parquet metadata cache in bytes. Zero means disabled.", 0) \ DECLARE(UInt64, parquet_metadata_cache_max_entries, DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES, "Maximum size of parquet metadata files cache in entries. Zero means disabled.", 0) \ DECLARE(Double, parquet_metadata_cache_size_ratio, DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the parquet metadata cache relative to the cache's total size.", 0) \ + DECLARE(String, puffin_files_cache_policy, DEFAULT_PUFFIN_FILES_CACHE_POLICY, "Puffin files cache policy name (SLRU or LRU).", 0) \ + DECLARE(UInt64, puffin_files_cache_size, DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE, "Maximum size of Puffin files cache in bytes. Zero means disabled.", 0) \ + DECLARE(UInt64, puffin_files_cache_max_entries, DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES, "Maximum size of Puffin files cache in entries. Zero means disabled.", 0) \ + DECLARE(Double, puffin_files_cache_size_ratio, DEFAULT_PUFFIN_FILES_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the Puffin files cache relative to the cache's total size.", 0) \ DECLARE(String, allowed_disks_for_table_engines, "", "List of disks allowed for use with Iceberg", 0) \ DECLARE(String, vector_similarity_index_cache_policy, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_POLICY, "Vector similarity index cache policy name.", 0) \ DECLARE(UInt64, vector_similarity_index_cache_size, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_MAX_SIZE, R"(Size of cache for vector similarity indexes. Zero means disabled. @@ -2010,6 +2015,10 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {"parquet_metadata_cache_size", {std::to_string(context->getParquetMetadataCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}); #endif + if (context->getPuffinFilesCache()) + changeable_settings.insert( + {"puffin_files_cache_size", + {std::to_string(context->getPuffinFilesCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}); /// `keeper_hosts` is not a regular config setting; it is derived from the `` config and follows /// it on config reload, so the live value diverges from the empty default stored in `ServerSettings`. diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 7f5f7a2a00ff..e42070ffa077 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5572,6 +5572,14 @@ Minimum time of delay between 2 background compaction operations. )", 0) \ DECLARE(Seconds, iceberg_compaction_data_cleanup, 60 * 60 * 3, R"( The time after which the data will be deleted. +)", 0) \ + DECLARE(Bool, use_puffin_files_cache, true, R"( +If turned on, Iceberg reads may utilize the Puffin files cache for parsed puffin file content such as deletion vectors. + +Possible values: + +- 0 - Disabled +- 1 - Enabled )", 0) \ DECLARE(Bool, use_query_cache, false, R"( If turned on, `SELECT` queries may utilize the [query cache](../query-cache.md). Parameters [enable_reads_from_query_cache](#enable_reads_from_query_cache) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index c2b62c209863..c1e28a04f695 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,6 +39,96 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.6.1.20001.altinityantalya", + { + {"max_insert_threads", 1, 0, "Changed the default from 1 (no parallel execution) to auto (0), which resolves to the number of CPU cores available to the server, reduced under memory pressure via `max_insert_threads_min_free_memory_per_thread`. This parallelizes `INSERT SELECT` by default. Set to 1 to restore the previous single-threaded behavior."}, + {"unique_key_probe_implementation", "auto", "auto", "New setting: selects the UNIQUE KEY probe implementation (currently only the simple baseline exists)"}, + {"s3_base", "", "", "New setting to specify the base URL for resolving relative URLs in the s3 table function and the S3 table engine."}, + {"use_query_condition_cache_for_top_k", false, false, "New setting to gate the query condition cache for `ORDER BY ... LIMIT n` (TopK) reads; disabled by default."}, + {"use_projection_index_in_read_pools", false, false, "New setting to drop mark ranges fully filtered out by a projection index before read tasks are created in MergeTree read pools."}, + {"allow_distinct_partitions_independently", false, true, "New setting to enable independent per-partition evaluation of `DISTINCT` when the partition expression is a deterministic function of the `DISTINCT` columns."}, + {"force_distinct_partitions_independently", false, false, "New setting to force independent per-partition evaluation of `DISTINCT` even when the cost heuristic would skip it."}, + {"max_number_of_partitions_for_independent_distinct", 128, 128, "New setting: maximal number of partitions to apply independent per-partition `DISTINCT`."}, + {"allow_lossy_numeric_supertype", false, false, "New setting that lets if/multiIf/coalesce/ifNull/array/map resolve all-numeric branches with no lossless common type (e.g. Decimal + Float64) to a numeric supertype (Float64, with possible precision loss), so the result can be aggregated. Independent of use_variant_as_common_type: with it off such branches previously raised NO_COMMON_TYPE, with it on they became a Variant; either way they now resolve to Float64."}, + {"throw_on_hive_partitioning_resolution_failure", false, true, "New setting to fail the query when Hive-style partitioning detection for an object storage table cannot list the storage, instead of running without the Hive partition columns."}, + {"allow_experimental_json_ast_dialect", false, false, "New setting to enable the `clickhouse_json` value of the `dialect` setting, which interprets queries as JSON ASTs (the output of `parseQueryToJSON`) instead of SQL text."}, + {"analyzer_compatibility_apply_final_to_all_joined_tables", false, false, "New setting on master (default false = the fixed behavior). The behavior flip itself is recorded under 26.6, and the introduction for backports to older release branches (with default true) under 26.4."}, + {"enable_parallel_single_level_merge", false, true, "New setting to parallelize the final merge of the single-level aggregation hash tables by splitting the key space into disjoint hash partitions that the threads merge independently."}, + {"ai_function_text_default_credentials", "", "", "New setting"}, + {"ai_function_embedding_default_credentials", "", "", "New setting"}, + {"ai_function_allow_insecure_endpoint", true, false, "AI functions now reject insecure (http) endpoints to remote hosts by default."}, + {"ai_function_max_api_calls_per_query", 0, 1000, "Bound outbound AI function HTTP calls per query by default (previously 0 - unlimited)."}, + {"join_runtime_filter_min_probe_rows", 0, 1000, "New setting to control minimum probe side size for installing JOIN runtime filters. It wasn't limited before, so previous value is 0 meaning always install."}, + {"optimize_trivial_count_with_sparsity_filter", false, true, "Promote to BETA and enable by default: serve `SELECT count() FROM t WHERE ` from the persisted per-column `num_defaults` / `num_rows` counters when `` partitions rows into defaults vs non-defaults. Requires the MergeTree setting `compute_exact_num_defaults_for_sparse_columns` (also enabled by default now)."}, + {"input_format_parquet_dictionary_filter_push_down", 0, 1024 * 1024, "New setting enabling Parquet row-group pruning based on dictionary page contents (reader v3). The value is the maximum dictionary page size in bytes for which the optimization applies; 0 (the previous behavior) disables it."}, + {"input_format_read_datetime_number_as_raw_value", true, false, "From 26.8, an unquoted number for a `DateTime`/`DateTime64` column in the `JSON` and `Values`/`Quoted` paths (and in `JSONExtract` and typed `JSON`) is a Unix timestamp in seconds, consistent with the `Values` format, `CAST` and `toDateTime64`. Set this to `true` (or `SET compatibility = '26.7'`) to restore the pre-26.8 behavior, where a bare unquoted integer fed to a `DateTime64` column was read as the raw scaled value (ticks). The tab-separated, CSV and other escaped/whole-text formats are not governed by this setting."}, + {"query_plan_short_circuit_constant_false_join", false, true, "New setting to short-circuit a JOIN with a constant-false ON condition so the non-contributing side is not read. previous_value=false so `compatibility` with versions before 26.8 restores the pre-existing behavior (no short-circuit)."}, + {"input_format_arrow_use_native_reader", true, true, "Obsolete setting, the native ClickHouse reader is now always used for the `Arrow` and `ArrowStream` formats (the Apache Arrow library-based reader has been removed)."}, + {"output_format_arrow_use_native_writer", true, true, "Obsolete setting, the native ClickHouse writer is now always used for the `Arrow` and `ArrowStream` formats (the Apache Arrow library-based writer has been removed)."}, + {"distributed_cache_min_inflight_bytes_to_discard_connection_on_seek", 0, 4 * 1024 * 1024, "New setting to drop and reopen a distributed cache connection on a seek when too many in-flight bytes would otherwise be discarded. Defaults to 4 MiB; 0 restores the previous behavior (always reuse the connection via the read range id)."}, + {"input_format_parquet_spatial_filter_push_down", false, true, "New setting: skip GeoParquet row groups and pages based on spatial predicates and bounding box statistics"}, + {"use_text_index_negative_tokens_cache", false, true, "New setting to cache absent text index tokens and avoid repeated dictionary lookups."}, + }); + addSettingsChanges(settings_changes_history, "26.7", + { + {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, + {"dictionary_lazy_load", "auto", "auto", "New setting overriding the server setting `dictionaries_lazy_load` for an individual dictionary."}, + {"discard_query_data", false, false, "New setting to skip sending query result rows to the client over the native TCP protocol."}, + {"optimize_trivial_count_with_sparsity_filter", false, false, "New (experimental) setting to serve `SELECT count() FROM t WHERE ` from per-column `num_defaults` / `num_rows` recorded in `serialization.json` when `` partitions rows into defaults vs non-defaults."}, + {"merge_tree_generic_exclusion_search_max_steps", 0, 0, "New setting to limit the number of steps of the generic exclusion search over the primary key index."}, + {"use_streaming_marks_compression", false, false, "New setting to compress marks into in-memory representation one block at a time (streaming) instead of materializing the full plain marks array, reducing peak memory during marks loading for compact parts with many substreams."}, + {"s3_validate_etag_on_read", false, true, "New setting to detect concurrent in-place overwrites of S3/GCS objects during a read by validating the GET response ETag against the listed one. previous_value=false so `compatibility` with versions before 26.7 restores the pre-existing behavior (no validation)."}, + {"dead_blobs_to_delay_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, + {"dead_blobs_to_throw_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, + {"input_format_csv_missing_nullable_as_empty_string", false, false, "New setting to read a missing value of `Nullable(String)` from CSV as an empty string instead of NULL."}, + {"use_legacy_to_time", true, false, "Use the new `toTime` function (converting values to the `Time` data type) by default instead of the legacy `toTime` (which is still available as `toTimeWithFixedDate`)."}, + {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, + {"parallel_replicas_plan_based", false, false, "New setting"}, + {"use_paimon_metadata_files_cache", false, false, "New setting to enable in-memory caching of parsed Paimon metadata files (manifest lists and manifests). For persistent Paimon table engines it must be enabled before metadata initialization; table functions evaluate it per query. Avoids repeated downloads and deserialization of metadata files from object storage on subsequent queries."}, + {"optimize_or_like_chain", false, true, "Enable by default: optimize OR chains of LIKE/ILIKE/match into multiSearchAny (pure-substring patterns) or multiMatchAny (other patterns, when Hyperscan/Vectorscan is permitted); when neither fast path applies the original OR chain is kept unchanged."}, + {"optimize_or_like_chain_min_patterns", 0, 10, "New setting controlling the minimum number of non-pure-substring LIKE/ILIKE/match branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiMatchAny. Shorter chains are kept as-is because the multiMatchAny (Hyperscan) rewrite only becomes faster than short-circuit OR evaluation from about nine branches."}, + {"optimize_or_like_chain_min_substrings", 0, 4, "New setting controlling the minimum number of pure-substring (%needle%) LIKE/ILIKE branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiSearchAny."}, + {"input_format_arrow_use_native_reader", false, true, "New setting to use the native ClickHouse reader for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, + {"input_format_orc_use_fast_decoder", true, true, "Obsolete setting, the native ClickHouse ORC decoder is now always used (the Apache Arrow-based ORC reader has been removed)."}, + {"output_format_arrow_use_native_writer", false, true, "New setting to use the native ClickHouse writer for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, + {"allow_minmax_index_for_json", true, false, "Forbid creating minmax skip index on JSON columns by default because the index serialization cannot handle heterogeneous Field values"}, + {"s3_allow_server_credentials_in_user_queries", true, false, "New setting to block S3 access from user SQL from resolving the server's own ambient credentials (environment/IMDS/IRSA/instance-profile/AWS-config-file/GCP-OAuth-metadata). Explicit role_arn-based STS assume-role is still allowed. The previous behavior (allowed) is restored with compatibility settings."}, + {"query_plan_merge_expression_into_join", false, true, "New setting. Allow to merge Expression step into JOIN step during join reordering optimization."}, + {"skip_unavailable_shards_mode", "unavailable_or_table_missing", "unavailable_or_table_missing", "New setting to control which exceptions from a remote shard are ignored when `skip_unavailable_shards` is enabled. The default matches the historical behavior: a shard whose table is missing is treated as unavailable."}, + {"use_text_index_tokens_cache", false, true, "Enabled the text index tokens cache globally."}, + {"use_text_index_header_cache", false, true, "Enabled the text index header cache globally."}, + {"optimize_aggregation_in_order_limit", false, true, "New setting to push the `LIMIT` into aggregation-in-order for early termination when the `ORDER BY` is a prefix of the `GROUP BY` sort description."}, + {"explain_query_plan_default", "legacy", "pretty", "From 26.7, `EXPLAIN PLAN` defaults to `actions=1, compact=1, pretty=1`. Set this to `legacy` to restore the pre-26.7 output."}, + {"format_geojson_validate_geometry", true, true, "New setting that controls whether the GeoJSON format enforces RFC 7946 geometry validity (minimum points per line and ring, ring closure, non-empty multi-geometries) when reading and writing"}, + {"use_partition_minmax_for_primary_key_pruning", false, true, "New setting to use the part's partition minmax to prune more granules during primary key analysis for `MergeTree` tables, when a primary key column is also an input column of the partition key."}, + {"allow_delta_lake_writes", false, false, "Added an alias for setting `allow_experimental_delta_lake_writes`, which was moved to Beta."}, + {"allow_experimental_delta_lake_writes", false, false, "Delta Lake writes were moved to Beta."}, + {"optimize_redundant_comparisons", false, true, "New setting to detect conflicting and redundant comparison conditions on the same expression within AND chains."}, + {"mysql_datatypes_support_level", "decimal,datetime64,date2Date32", "decimal,datetime64,date2Date32,geometry", "Map MySQL's concrete spatial types (LINESTRING, POLYGON, MULTILINESTRING, MULTIPOLYGON, MULTIPOINT) and the generic GEOMETRY type to the corresponding ClickHouse geometric types by default. The generic GEOMETRY column maps to the umbrella Geometry type; reading a value whose subtype has no ClickHouse counterpart (GEOMETRYCOLLECTION) throws at read time."}, + {"snappy_mode", "basic", "basic", "New setting to control the wire format used for snappy compression in generic file/URL I/O. The default `basic` preserves backward-compatible Hadoop snappy block format reads; HTTP `Content-Encoding: snappy` always uses the framing format independently of this setting."}, + {"compile_regular_expressions", false, true, "New setting to enable JIT compilation of simple regular expressions in functions like `match` and `extract`."}, + {"min_count_to_compile_regular_expression", 3, 3, "New setting controlling how many times a regular expression must be used before it is JIT-compiled."}, + {"allow_aggregate_partitions_independently", false, true, "Enable independent per-partition aggregation by default when the partition key suits the GROUP BY key. The existing runtime heuristics in `ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePortForAggregation` already skip the optimization when the partition layout is unfavorable (too few partitions, too many partitions, or significantly skewed partition sizes), so enabling the setting is safe in the cases where it would otherwise be a no-op."}, + {"text_index_lazy_intersection_density_threshold", 0.2, 0.2, "Renamed from `text_index_density_threshold` (kept as an alias); selects the posting list intersection algorithm in lazy posting list apply mode."}, + {"allow_experimental_text_index_lazy_apply", false, true, "Lazy posting list apply mode for the text index is no longer experimental; the setting is now obsolete and has no effect (lazy mode is selected via `text_index_posting_list_apply_mode = 'lazy'`)."}, + {"allow_experimental_url_wildcard_from_index_pages", false, false, "New setting to enable expanding wildcards in the `url` table function by listing HTTP index pages."}, + {"url_wildcard_max_directories_to_read", 100000, 100000, "New setting to limit the number of directories read when expanding wildcards in the `url` table function."}, + {"allow_experimental_eval_table_function", false, false, "New setting to enable the experimental table function `eval`."}, + {"output_format_csv_header_serialize_tuple_into_separate_columns", false, true, "New setting. When output_format_csv_serialize_tuple_into_separate_columns is enabled, the CSVWithNames/CSVWithNamesAndTypes header now flattens Tuple columns into their leaf fields so the header width matches the data. Set to false to restore the previous single-name header."}, + {"enable_join_runtime_filters_index_analysis", false, false, "New setting to enable join filtering using dynamic index analysis"}, + {"vector_search_use_quantized_codes", false, false, "New setting to opt into the two-stage approximate vector-search optimization over a Quantize(...) column codec; queries stay exact by default."}, + {"reader_executor_use_long_connections", false, false, "New experimental ReaderExecutor setting (off by default): reuse a held source connection across sequential windows."}, + {"reader_executor_min_bytes_for_seek", 2097152, 2097152, "New experimental ReaderExecutor setting: forward-gap bound for bridging on a held source connection."}, + {"reader_executor_max_tail_for_drain", 1048576, 1048576, "New experimental ReaderExecutor setting: drain bound for completing a dropped long connection."}, + {"precise_float_parsing", false, true, "Use the precise (closest-representable) float parsing algorithm by default, now that it is faster than the previous fast algorithm. Set to false to restore the pre-26.7 fast-but-less-accurate parsing in conversion functions."}, + {"optimize_and_compare_chain_max_hash_work", 0, 5'000'000, "New setting that bounds the work of the `optimize_and_compare_chain` optimization (measured in query-tree nodes hashed) so it cannot dominate analysis of queries with very many or very large `AND`-chains of comparisons. The previous value `0` (unlimited) reproduces the pre-26.7 behavior where the optimization was uncapped, so `compatibility` set to an earlier version keeps deriving transitive predicates without a budget. Set to `0` to disable the budget."}, + {"iceberg_manifest_min_count_to_compact", 30, 30, "New setting to control manifest compaction for Iceberg tables."}, + {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, + {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, + {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, + {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, + }); + addSettingsChanges(settings_changes_history, "26.6", { {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the new analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, @@ -145,6 +235,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"allow_experimental_cleanup_old_data_files_compaction", false, false, "New setting"}, {"iceberg_compaction_data_cleanup", 60 * 60 * 3, 60 * 60 * 3, "New setting"}, {"allow_rank_dense_rank_arguments", true, false, "New setting. Before 26.5, the `RANK` and `DENSE_RANK` window functions silently ignored any provided arguments (equivalent to `allow_rank_dense_rank_arguments = 1`). From 26.5, they reject arguments by default with `NUMBER_OF_ARGUMENTS_DOESNT_MATCH` because per SQL standard these functions take zero arguments. Set this to `1` to restore the legacy behavior."}, + {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, }); addSettingsChanges(settings_changes_history, "26.4", { diff --git a/src/Interpreters/ClusterFunctionReadTask.cpp b/src/Interpreters/ClusterFunctionReadTask.cpp index f3cfad1fba31..57f15dd0baa4 100644 --- a/src/Interpreters/ClusterFunctionReadTask.cpp +++ b/src/Interpreters/ClusterFunctionReadTask.cpp @@ -82,6 +82,46 @@ void ClusterFunctionReadTaskResponse::serialize(WriteBuffer & out, size_t worker { auto protocol_version = std::min(static_cast(worker_protocol_version), static_cast(DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION)); + + /// Fail closed: downgrading would omit deletion / selection vectors and return deleted rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS + && hasNonEmptyExcludedRows(data_lake_metadata)) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `excluded_rows`, which is required for distributed " + "reads with deletion vectors / selection vectors (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS); + } + + /// Fail closed: protocol < 3 omits `iceberg_info`, so workers rebuild a plain `ObjectInfo` and + /// skip equality / parquet position delete transforms — returning deleted rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA + && iceberg_info.has_value() + && (!iceberg_info->position_deletes_objects.empty() || !iceberg_info->equality_deletes_objects.empty())) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `iceberg_info` with equality / position deletes " + "(minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + } + + /// Fail closed: protocol < 4 omits `file_bucket_info`, so each bucket task becomes a full-file + /// read and bucket-split cluster queries return duplicated rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO + && file_bucket_info) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `file_bucket_info`, which is required for " + "distributed bucket-split reads (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + } + writeVarUInt(protocol_version, out); writeStringBinary(path, out); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 1edce5d06b45..4d66cc0e7040 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -586,6 +587,7 @@ struct ContextSharedPart : boost::noncopyable #if USE_PARQUET mutable ParquetMetadataCachePtr parquet_metadata_cache TSA_GUARDED_BY(mutex); /// Cache of deserialized parquet metadata files. #endif + mutable PuffinFilesCachePtr puffin_files_cache TSA_GUARDED_BY(mutex); /// Cache of parsed puffin file content. AsynchronousMetrics * asynchronous_metrics TSA_GUARDED_BY(mutex) = nullptr; /// Points to asynchronous metrics mutable PageCachePtr page_cache TSA_GUARDED_BY(mutex); /// Userspace page cache. ProcessList process_list; /// Executing queries at the moment. @@ -4755,6 +4757,52 @@ void Context::clearParquetMetadataCache() const } #endif +void Context::setPuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_entries, double size_ratio) +{ + std::lock_guard lock(shared->mutex); + + if (shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache has been already created."); + + shared->puffin_files_cache = std::make_shared(cache_policy, max_size_in_bytes, max_entries, size_ratio); +} + +void Context::updatePuffinFilesCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) +{ + std::lock_guard lock(shared->mutex); + + if (!shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache was not created yet."); + + size_t size = config.getUInt64("puffin_files_cache_size", DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE); + size_t max_entries = config.getUInt64("puffin_files_cache_max_entries", DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES); + if (size > max_cache_size) + { + size = max_cache_size; + LOG_DEBUG(shared->log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(size)); + } + shared->puffin_files_cache->setMaxSizeInBytes(size); + shared->puffin_files_cache->setMaxCount(max_entries); +} + +std::shared_ptr Context::getPuffinFilesCache() const +{ + SharedLockGuard lock(shared->mutex); + + if (!shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache was not created yet."); + return shared->puffin_files_cache; +} + +void Context::clearPuffinFilesCache() const +{ + auto cache = getPuffinFilesCache(); + + /// Clear the cache without holding context mutex to avoid blocking context for a long time + if (cache) + cache->clear(); +} + void Context::setQueryConditionCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e939d24e0e86..ae290ef0c99c 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -109,6 +109,7 @@ class MMappedFileCache; class UncompressedCache; class IcebergMetadataFilesCache; class ParquetMetadataCache; +class PuffinFilesCache; class VectorSimilarityIndexCache; class TextIndexTokensCache; class TextIndexHeaderCache; @@ -1508,6 +1509,11 @@ class Context: public ContextData, public std::enable_shared_from_this void clearParquetMetadataCache() const; #endif + void setPuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_entries, double size_ratio); + void updatePuffinFilesCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); + std::shared_ptr getPuffinFilesCache() const; + void clearPuffinFilesCache() const; + void setAllowedDisksForTableEngines(std::unordered_set && allowed_disks_) { allowed_disks = std::move(allowed_disks_); } const std::unordered_set & getAllowedDisksForTableEngines() const { return allowed_disks; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 8220a1a82db6..0dc048f54def 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -481,6 +481,10 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet"); #endif + case Type::CLEAR_PUFFIN_FILES_CACHE: + getContext()->checkAccess(AccessType::SYSTEM_DROP_PUFFIN_FILES_CACHE); + system_context->clearPuffinFilesCache(); + break; case Type::CLEAR_PRIMARY_INDEX_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_PRIMARY_INDEX_CACHE); system_context->clearPrimaryIndexCache(); @@ -2475,6 +2479,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::CLEAR_ICEBERG_METADATA_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: case Type::CLEAR_PARQUET_METADATA_CACHE: + case Type::CLEAR_PUFFIN_FILES_CACHE: case Type::CLEAR_PRIMARY_INDEX_CACHE: case Type::CLEAR_MMAP_CACHE: case Type::CLEAR_QUERY_CONDITION_CACHE: diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index c4ee97348899..e5de3e8e8cf5 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -604,6 +604,7 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti case Type::CLEAR_ICEBERG_METADATA_CACHE: case Type::CLEAR_PARQUET_METADATA_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: + case Type::CLEAR_PUFFIN_FILES_CACHE: case Type::RESET_COVERAGE: case Type::RESTART_REPLICAS: case Type::JEMALLOC_PURGE: diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index adf76d898d42..814e76989893 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -44,6 +44,7 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster CLEAR_COMPILED_EXPRESSION_CACHE, CLEAR_ICEBERG_METADATA_CACHE, CLEAR_PARQUET_METADATA_CACHE, + CLEAR_PUFFIN_FILES_CACHE, CLEAR_FILESYSTEM_CACHE, CLEAR_DISTRIBUTED_CACHE, CLEAR_DISK_METADATA_CACHE, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 07b79465ea04..b2ed7392ee6c 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -285,6 +285,8 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & {"DROP COMPILED EXPRESSION CACHE", Type::CLEAR_COMPILED_EXPRESSION_CACHE}, {"DROP ICEBERG METADATA CACHE", Type::CLEAR_ICEBERG_METADATA_CACHE}, {"DROP PARQUET METADATA CACHE", Type::CLEAR_PARQUET_METADATA_CACHE}, + {"DROP PUFFIN FILES CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, + {"DROP PUFFIN_FILES_CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, {"DROP FILESYSTEM CACHE", Type::CLEAR_FILESYSTEM_CACHE}, {"DROP DISTRIBUTED CACHE", Type::CLEAR_DISTRIBUTED_CACHE}, {"DROP DISK METADATA CACHE", Type::CLEAR_DISK_METADATA_CACHE}, diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index d94b6ae42b81..d80c5e259a87 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -320,6 +321,46 @@ void Reader::getHyperrectangleForRowGroup(const parq::RowGroup * meta, Hyperrect } } +std::vector buildRowGroupGlobalOffsets(const parq::FileMetaData & file_metadata) +{ + if (file_metadata.num_rows < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet file has negative row count: {}", file_metadata.num_rows); + + const size_t num_row_groups = file_metadata.row_groups.size(); + std::vector global_offsets(num_row_groups + 1, 0); + UInt64 total_rows = 0; + + for (size_t i = 0; i < num_row_groups; ++i) + { + const Int64 num_rows = file_metadata.row_groups[i].num_rows; + if (num_rows < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet row group {} has negative row count: {}", i, num_rows); + + UInt64 next_total = 0; + if (common::addOverflow(total_rows, static_cast(num_rows), next_total)) + { + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet row group row counts overflow when computing global offsets (at row group {})", + i); + } + + total_rows = next_total; + global_offsets[i + 1] = static_cast(total_rows); + } + + if (total_rows != static_cast(file_metadata.num_rows)) + { + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Sum of Parquet row group row counts ({}) does not match FileMetaData.num_rows ({})", + total_rows, + file_metadata.num_rows); + } + + return global_offsets; +} + void Reader::prefilterAndInitRowGroups(const std::optional> & row_groups_to_read) { extended_sample_block = *sample_block; @@ -375,19 +416,15 @@ void Reader::prefilterAndInitRowGroups(const std::optional global_offsets = buildRowGroupGlobalOffsets(file_metadata); for (size_t row_group_idx = 0; row_group_idx < file_metadata.row_groups.size(); ++row_group_idx) { const auto * meta = &file_metadata.row_groups[row_group_idx]; - if (meta->num_rows < 0) - throw Exception(ErrorCodes::INCORRECT_DATA, "Row group {} has negative row count: {}", row_group_idx, meta->num_rows); if (meta->num_rows == 0) continue; /// Empty row groups are valid in Parquet; skip them. if (meta->columns.size() != total_primitive_columns_in_file) throw Exception(ErrorCodes::INCORRECT_DATA, "Row group {} has unexpected number of columns: {} != {}", row_group_idx, meta->columns.size(), total_primitive_columns_in_file); - total_rows += size_t(meta->num_rows); // before potentially skipping the row group - Hyperrectangle hyperrectangle(extended_sample_block.columns(), Range::createWholeUniverse()); if (options.format.parquet.filter_push_down && format_filter_info->key_condition) { @@ -401,7 +438,7 @@ void Reader::prefilterAndInitRowGroups(const std::optionalcontains(row_group_idx); row_group.row_group_idx = row_group_idx; - row_group.start_global_row_idx = total_rows - size_t(meta->num_rows); + row_group.start_global_row_idx = global_offsets[row_group_idx]; row_group.columns.resize(primitive_columns.size()); row_group.hyperrectangle = std::move(hyperrectangle); diff --git a/src/Processors/Formats/Impl/Parquet/Reader.h b/src/Processors/Formats/Impl/Parquet/Reader.h index 53b598aa5524..53dccefb1eed 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.h +++ b/src/Processors/Formats/Impl/Parquet/Reader.h @@ -549,4 +549,8 @@ struct Reader void readRowsInPage(size_t end_row_idx, ColumnSubchunk & subchunk, ColumnChunk & column, const PrimitiveColumnInfo & column_info, const RowSubgroup * row_subgroup = nullptr); }; +/// Prefix offsets of row groups in the file; result size is `row_groups.size() + 1`. +/// Throws `INCORRECT_DATA` on negative counts, size overflow, or mismatch with `FileMetaData.num_rows`. +std::vector buildRowGroupGlobalOffsets(const parq::FileMetaData & file_metadata); + } diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index edbf421ccbeb..918a16378fae 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -25,6 +25,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } static Parquet::ReadOptions convertReadOptions(const FormatSettings & format_settings) @@ -120,23 +121,68 @@ parquet::format::FileMetaData ParquetV3BlockInputFormat::getFileMetadata(Parquet } } -Chunk ParquetV3BlockInputFormat::read() +void ParquetV3BlockInputFormat::prepareNeedOnlyCountRowGroups(const parquet::format::FileMetaData & file_metadata) { - if (need_only_count) + const std::vector global_offsets = Parquet::buildRowGroupGlobalOffsets(file_metadata); + const size_t num_row_groups = file_metadata.row_groups.size(); + + need_only_count_row_groups.clear(); + need_only_count_next = 0; + + if (!buckets_to_read) { - if (reported_count) - return {}; + /// Emit one span per row group (not a single whole-file span). Bounds peak memory when a + /// downstream transform (e.g. DeletionVectorTransform) still walks chunk rows, and matches + /// the bucketed need-only-count shape. + for (size_t row_group_id = 0; row_group_id < num_row_groups; ++row_group_id) + { + const size_t num_rows = static_cast(file_metadata.row_groups[row_group_id].num_rows); + if (num_rows == 0) + continue; + + need_only_count_row_groups.push_back( + {.row_num_offset = global_offsets[row_group_id], .num_rows = num_rows}); + } + return; + } + + for (size_t row_group_id : buckets_to_read->row_group_ids) + { + if (row_group_id >= num_row_groups) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet bucket row group {} is out of range (file has {} row groups)", + row_group_id, + num_row_groups); + + const size_t num_rows = static_cast(file_metadata.row_groups[row_group_id].num_rows); + if (num_rows == 0) + continue; - /// Don't init Reader and ReadManager if we only need file metadata. - Parquet::Prefetcher temp_prefetcher; - temp_prefetcher.init(in, read_options, parser_shared_resources); - parquet::format::FileMetaData file_metadata = getFileMetadata(temp_prefetcher); + need_only_count_row_groups.push_back( + {.row_num_offset = global_offsets[row_group_id], .num_rows = num_rows}); + } +} +Chunk ParquetV3BlockInputFormat::read() +{ + if (need_only_count) + { + if (!need_only_count_prepared) + { + /// Don't init Reader and ReadManager if we only need file metadata. + Parquet::Prefetcher temp_prefetcher; + temp_prefetcher.init(in, read_options, parser_shared_resources); + prepareNeedOnlyCountRowGroups(getFileMetadata(temp_prefetcher)); + need_only_count_prepared = true; + } - auto chunk = getChunkForCount(size_t(file_metadata.num_rows)); - chunk.getChunkInfos().add(std::make_shared(0)); + if (need_only_count_next >= need_only_count_row_groups.size()) + return {}; - reported_count = true; + const auto & row_group = need_only_count_row_groups[need_only_count_next++]; + auto chunk = getChunkForCount(row_group.num_rows); + chunk.getChunkInfos().add(std::make_shared(row_group.row_num_offset)); return chunk; } @@ -199,6 +245,9 @@ void ParquetV3BlockInputFormat::resetParser() reader.reset(); } previous_block_missing_values.clear(); + need_only_count_prepared = false; + need_only_count_next = 0; + need_only_count_row_groups.clear(); IInputFormat::resetParser(); } diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h index 87a5192ae641..2c5baf00ea92 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h @@ -81,12 +81,21 @@ class ParquetV3BlockInputFormat final : public IInputFormat std::mutex reader_mutex; std::optional reader; - bool reported_count = false; // if need_only_count + bool need_only_count_prepared = false; + size_t need_only_count_next = 0; + + struct NeedOnlyCountRowGroup + { + size_t row_num_offset = 0; + size_t num_rows = 0; + }; + std::vector need_only_count_row_groups; BlockMissingValues previous_block_missing_values; size_t previous_approx_bytes_read_for_chunk = 0; void initializeIfNeeded(); + void prepareNeedOnlyCountRowGroups(const parquet::format::FileMetaData & file_metadata); std::shared_ptr buckets_to_read; parquet::format::FileMetaData getFileMetadata(Parquet::Prefetcher & prefetcher) const; diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp index e742a400410d..509cbe93de00 100644 --- a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp @@ -1,12 +1,6 @@ -#include #include -#include -#include #include #include -#include -#include -#include #include #include #include @@ -18,22 +12,15 @@ #include #include #include +#include #include #include -#include -#include -#include -#include -#include -#include #include -#include #include -#include #include #include - -#include +#include +#include namespace DB { @@ -41,28 +28,16 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int LZ4_DECODER_FAILED; } namespace { +/// Puffin footer parsing (magic, LZ4 payload, blob metadata JSON) lives in +/// `Storages/ObjectStorage/DataLakes/PuffinFile` and is shared with the Iceberg DV loader. +/// DV blob / materialization ceilings live in `PuffinDeletionVectorReader.h` (`PUFFIN_DV_MAX_*`). + constexpr UInt8 PUFFIN_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; -constexpr UInt8 PUFFIN_FOOTER_COMPRESSED_FLAG = 0x01; -constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; -constexpr size_t PUFFIN_FOOTER_LZ4_MAX_RATIO = 255; -constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; -constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; -constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; -constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; -constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; -constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; -constexpr const char * PUFFIN_DELETION_VECTOR_BLOB_TYPE = "deletion-vector-v1"; - -UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) -{ - return (static_cast(key) << 32) | static_cast(sub_position); -} void checkMagic(const UInt8 * p, const char * context) { @@ -70,412 +45,6 @@ void checkMagic(const UInt8 * p, const char * context) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin magic ({})", context); } -void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t blob_region_end, size_t blob_index) -{ - if (offset < static_cast(sizeof(PUFFIN_MAGIC)) || length < 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); - - Int64 end = 0; - if (common::addOverflow(offset, length, end)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); - - if (static_cast(end) > blob_region_end) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); -} - -void validatePuffinFooterFlags(const UInt8 flags[4]) -{ - if (flags[1] != 0 || flags[2] != 0 || flags[3] != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); - if ((flags[0] & ~PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); -} - -String decompressPuffinFooterPayload(const char * data, size_t size) -{ - LZ4F_dctx * dctx = nullptr; - size_t ret = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); - if (LZ4F_isError(ret)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to create LZ4 decompression context: {}", LZ4F_getErrorName(ret)); - - struct DecompressionContextGuard - { - LZ4F_dctx * dctx; - explicit DecompressionContextGuard(LZ4F_dctx * dctx_) : dctx(dctx_) { } - ~DecompressionContextGuard() { LZ4F_freeDecompressionContext(dctx); } - }; - DecompressionContextGuard guard(dctx); - - LZ4F_frameInfo_t frame_info{}; - size_t header_size = size; - ret = LZ4F_getFrameInfo(dctx, &frame_info, data, &header_size); - if (LZ4F_isError(ret)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to read LZ4 frame info for Puffin footer: {}", LZ4F_getErrorName(ret)); - - const char * src = data + header_size; - size_t src_remaining = size - header_size; - - size_t max_by_ratio = 0; - if (common::mulOverflow(size, PUFFIN_FOOTER_LZ4_MAX_RATIO, max_by_ratio)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer compressed payload is too large"); - - if (frame_info.contentSize == 0) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame must declare content size"); - - const size_t max_decompressed_size = std::min(max_by_ratio, PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); - if (frame_info.contentSize > max_decompressed_size) - { - if (max_decompressed_size == PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 content size {} exceeds absolute decompression limit {}", - frame_info.contentSize, - PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); - - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 content size {} exceeds decompression limit {}", - frame_info.contentSize, - max_decompressed_size); - } - - String result; - result.resize(static_cast(frame_info.contentSize)); - - size_t dst_offset = 0; - while (true) - { - size_t src_read = src_remaining; - size_t dst_write = result.size() - dst_offset; - ret = LZ4F_decompress(dctx, result.data() + dst_offset, &dst_write, src, &src_read, nullptr); - if (LZ4F_isError(ret)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to decompress Puffin footer: {}", LZ4F_getErrorName(ret)); - - src += src_read; - src_remaining -= src_read; - dst_offset += dst_write; - - if (ret == 0) - break; - - if (dst_write == 0 && src_read == 0) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame is incomplete"); - - if (dst_offset == result.size()) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer decompressed size exceeds declared content size {}", - frame_info.contentSize); - } - - if (dst_offset != frame_info.contentSize) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 decompressed size {} does not match content size {}", - dst_offset, - frame_info.contentSize); - - if (src_remaining != 0) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 frame has {} trailing bytes", - src_remaining); - - result.resize(dst_offset); - return result; -} - -void requireBlobMetadataField(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - if (!blob_obj->has(field_name) || blob_obj->isNull(field_name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field '{}'", blob_index, field_name); -} - -bool isJSONInteger(const Poco::Dynamic::Var & value) -{ - return value.isInteger() && !value.isBoolean(); -} - -std::optional tryJSONIntegerAsInt64(const Poco::Dynamic::Var & value) -{ - if (value.isInteger() && !value.isSigned()) - { - const auto as_uint64 = value.convert(); - if (as_uint64 > static_cast(std::numeric_limits::max())) - return std::nullopt; - return static_cast(as_uint64); - } - - return value.convert(); -} - -Int64 requireBlobMetadataInt64(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - requireBlobMetadataField(blob_obj, field_name, blob_index); - const Poco::Dynamic::Var & value = blob_obj->get(field_name); - if (!isJSONInteger(value)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be an integer", blob_index, field_name); - - auto as_int64 = tryJSONIntegerAsInt64(value); - if (!as_int64) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' is out of Int64 range", blob_index, field_name); - return *as_int64; -} - -Int32 requireBlobMetadataFieldsElementInt32(const Poco::Dynamic::Var & value, size_t blob_index, size_t field_index) -{ - if (!isJSONInteger(value)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] must be an integer", blob_index, field_index); - - auto as_int64 = tryJSONIntegerAsInt64(value); - if (!as_int64) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int64 range", blob_index, field_index); - - if (*as_int64 < std::numeric_limits::min() || *as_int64 > std::numeric_limits::max()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int32 range", blob_index, field_index); - - return static_cast(*as_int64); -} - -String requireJSONStringValue(const Poco::Dynamic::Var & value, size_t blob_index, const char * field_name) -{ - if (!value.isString()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); - return value.extract(); -} - -String requireBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - requireBlobMetadataField(blob_obj, field_name, blob_index); - return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); -} - -void requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) -{ - static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; - for (const char * key : required_properties) - { - auto it = blob.properties.find(key); - if (it == blob.properties.end() || it->second.empty()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 missing required property '{}'", - blob_index, - key); - } - - UInt64 cardinality = 0; - if (!tryParse(cardinality, blob.properties.at("cardinality"))) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", - blob_index); -} - -void parseStringValuedProperties( - const Poco::JSON::Object::Ptr & props_obj, - std::map * out, - bool for_blob, - size_t blob_index) -{ - for (const auto & [key, val] : *props_obj) - { - if (!val.isString()) - { - if (for_blob) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: property '{}' must be a string", blob_index, key); - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer property '{}' must be a string", key); - } - if (out) - out->emplace(key, val.extract()); - } -} - -void parseBlobProperties(const Poco::JSON::Object::Ptr & blob_obj, PuffinBlob & blob, size_t blob_index, bool required) -{ - if (!blob_obj->has("properties")) - { - if (required) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field 'properties'", blob_index); - return; - } - - if (blob_obj->isNull("properties")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); - - auto props_obj = blob_obj->getObject("properties"); - if (!props_obj) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); - - parseStringValuedProperties(props_obj, &blob.properties, /*for_blob=*/true, blob_index); -} - -void validateFileMetadataProperties(const Poco::JSON::Object::Ptr & footer_obj) -{ - if (!footer_obj->has("properties")) - return; - - if (footer_obj->isNull("properties")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); - - auto props_obj = footer_obj->getObject("properties"); - if (!props_obj) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); - - parseStringValuedProperties(props_obj, /*out=*/nullptr, /*for_blob=*/false, /*blob_index=*/0); -} - -std::vector parseFooterJSON(const String & footer_json, size_t blob_region_end) -{ - Poco::JSON::Parser parser; - Poco::Dynamic::Var root; - try - { - root = parser.parse(footer_json); - } - catch (const Exception &) - { - throw; - } - catch (const Poco::Exception & e) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Puffin footer JSON: {}", e.displayText()); - } - - if (root.type() != typeid(Poco::JSON::Object::Ptr)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer JSON must be an object"); - - auto obj = root.extract(); - - validateFileMetadataProperties(obj); - - if (!obj->has("blobs") || obj->isNull("blobs")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer is missing required field 'blobs'"); - - auto blobs_arr = obj->getArray("blobs"); - if (!blobs_arr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'blobs' must be an array"); - - std::vector blobs; - for (size_t i = 0; i < blobs_arr->size(); ++i) - { - auto blob_obj = blobs_arr->getObject(static_cast(i)); - if (!blob_obj) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: must be an object", i); - - PuffinBlob blob; - - blob.type = requireBlobMetadataString(blob_obj, "type", i); - blob.snapshot_id = requireBlobMetadataInt64(blob_obj, "snapshot-id", i); - blob.sequence_number = requireBlobMetadataInt64(blob_obj, "sequence-number", i); - blob.offset = requireBlobMetadataInt64(blob_obj, "offset", i); - blob.length = requireBlobMetadataInt64(blob_obj, "length", i); - - if (blob.type == PUFFIN_DELETION_VECTOR_BLOB_TYPE) - { - if (blob_obj->has("compression-codec")) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 must omit 'compression-codec'", - i); - - parseBlobProperties(blob_obj, blob, i, /*required=*/true); - requireDeletionVectorV1Properties(blob, i); - } - else - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: unsupported blob type '{}', only '{}' is supported", - i, - blob.type, - PUFFIN_DELETION_VECTOR_BLOB_TYPE); - } - - requireBlobMetadataField(blob_obj, "fields", i); - auto fields_arr = blob_obj->getArray("fields"); - if (!fields_arr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'fields' must be an array", i); - - for (size_t j = 0; j < fields_arr->size(); ++j) - blob.fields.push_back(requireBlobMetadataFieldsElementInt32(fields_arr->get(static_cast(j)), i, j)); - - validatePuffinBlobBounds(blob.offset, blob.length, blob_region_end, i); - - blobs.push_back(std::move(blob)); - } - return blobs; -} - -std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size) -{ - if (file_size < 16) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); - - seekable.seek(0, SEEK_SET); - char magic_buf[4]; - seekable.readStrict(magic_buf, 4); - checkMagic(reinterpret_cast(magic_buf), "header"); - - seekable.seek(static_cast(file_size - PUFFIN_FOOTER_TRAILER_SIZE), SEEK_SET); - Int32 footer_length_signed = 0; - readBinaryLittleEndian(footer_length_signed, seekable); - - UInt8 flags[4] = {}; - seekable.readStrict(reinterpret_cast(flags), sizeof(flags)); - - char trailing_buf[4]; - seekable.readStrict(trailing_buf, 4); - checkMagic(reinterpret_cast(trailing_buf), "trailing"); - validatePuffinFooterFlags(flags); - - if (footer_length_signed <= 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); - - UInt64 footer_trailer_size = 0; - if (common::addOverflow( - static_cast(footer_length_signed), - static_cast(PUFFIN_FOOTER_TRAILER_SIZE), - footer_trailer_size) - || footer_trailer_size > file_size) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); - - const size_t footer_length = static_cast(footer_length_signed); - const size_t payload_start = file_size - PUFFIN_FOOTER_TRAILER_SIZE - footer_length; - /// Footer layout is Magic | FooterPayload | …; require a distinct footer-open Magic after the - /// file header Magic (payload_start == 4 would reuse the header Magic when blobs is empty). - if (payload_start < 2 * sizeof(PUFFIN_MAGIC)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); - - if (footer_length > PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin footer payload size {} exceeds absolute limit {}", - footer_length, - PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); - - seekable.seek(static_cast(payload_start - sizeof(PUFFIN_MAGIC)), SEEK_SET); - char leading_magic_buf[4]; - seekable.readStrict(leading_magic_buf, 4); - checkMagic(reinterpret_cast(leading_magic_buf), "footer"); - - String footer_payload(footer_length, '\0'); - seekable.seek(static_cast(payload_start), SEEK_SET); - seekable.readStrict(footer_payload.data(), footer_length); - - String footer_json; - if ((flags[0] & PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) - footer_json = decompressPuffinFooterPayload(footer_payload.data(), footer_payload.size()); - else - footer_json = std::move(footer_payload); - - const size_t blob_region_end = payload_start - sizeof(PUFFIN_MAGIC); - return parseFooterJSON(footer_json, blob_region_end); -} - PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) { PuffinFooter result; @@ -483,27 +52,27 @@ PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) auto * seekable = dynamic_cast(&buf); auto file_size_opt = tryGetFileSizeFromReadBuffer(buf); + /// Pipes/FIFOs are SeekableReadBuffer subclasses but fstat reports size 0; require a real + /// regular file before trusting seek+size (same pattern as ORC/Arrow). Also honor + /// `input_format_allow_seeks` via FormatSettings.seekable_read. if (seekable_read && seekable && seekable->checkIfActuallySeekable() && file_size_opt) { - result.blobs = readPuffinFooterFromSeekable(*seekable, *file_size_opt); + result.blobs = readPuffinFooterBlobsFromSeekable(*seekable, *file_size_opt); } else { + /// Fail fast on format mismatch before buffering the rest of a (possibly huge) stream — + /// same pattern as `asArrowFileLoadIntoMemory`. result.data.resize(sizeof(PUFFIN_MAGIC)); const size_t magic_read = buf.read(reinterpret_cast(result.data.data()), sizeof(PUFFIN_MAGIC)); if (magic_read < sizeof(PUFFIN_MAGIC)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); checkMagic(result.data.data(), "header"); - std::vector tmp(DEFAULT_BLOCK_SIZE); - while (!buf.eof()) - { - size_t n = buf.read(reinterpret_cast(tmp.data()), tmp.size()); - result.data.insert(result.data.end(), tmp.data(), tmp.data() + n); - } + appendReadBufferWithAbsoluteSizeLimit(buf, result.data, PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE); ReadBufferFromMemory mem_buf(result.data.data(), result.data.size()); - result.blobs = readPuffinFooterFromSeekable(mem_buf, result.data.size()); + result.blobs = readPuffinFooterBlobsFromSeekable(mem_buf, result.data.size()); } return result; @@ -553,6 +122,10 @@ void readDeletionVectorEnvelopePrefix( seekable->readStrict(reinterpret_cast(header), 8); } +/// Validate absolute size and the first 8 envelope bytes (combined_length + magic) before allocating +/// `blob.length`. CRC / roaring deserialize run later in `deserializeDeletionVectorV1Blob` after a +/// bounded read — we intentionally do not stream roaring here (larger redesign; CRC coverage is +/// over magic+vector). String readDeletionVectorBlobBytes( const PuffinBlob & blob, ReadBuffer & buf, const std::vector & data, bool seekable_read) { @@ -571,168 +144,11 @@ String readDeletionVectorBlobBytes( UInt8 header[8]; readDeletionVectorEnvelopePrefix(blob, buf, data, seekable_read, header); - - ReadBufferFromMemory header_buf(reinterpret_cast(header), sizeof(header)); - UInt32 combined_length = 0; - readBinaryBigEndian(combined_length, header_buf); - if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); - - if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); - - UInt64 expected_blob_size = 0; - if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); - - if (static_cast(blob.length) != expected_blob_size) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector blob size {} does not match combined length {}", - blob.length, - combined_length); + validateDeletionVectorEnvelope(header, blob.length); return readPuffinBlobBytes(blob, buf, data, seekable_read); } -roaring::Roaring readRoaringPortableSafe(const char * data, size_t size, Int32 key) -{ - try - { - return roaring::Roaring::readSafe(data, size); - } - catch (const std::exception & e) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to deserialize deletion vector roaring bitmap at key {}: {}", key, e.what()); - } -} - -void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_cardinality, ColumnUInt64 & positions) -{ - if (bytes.size() < sizeof(Int64)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is too small"); - - const char * ptr = bytes.data(); - size_t remaining = bytes.size(); - - /// Iceberg deletion-vector roaring layout stores count and keys as little-endian. - const Int64 bitmap_count = unalignedLoadLittleEndian(ptr); - ptr += sizeof(Int64); - remaining -= sizeof(Int64); - - if (bitmap_count < 0 || bitmap_count > std::numeric_limits::max()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap count: {}", bitmap_count); - - Int32 last_key = -1; - Int32 remaining_count = static_cast(bitmap_count); - UInt64 running_cardinality = 0; - - while (remaining_count > 0) - { - if (remaining < sizeof(Int32)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is truncated while reading key"); - - const Int32 key = unalignedLoadLittleEndian(ptr); - ptr += sizeof(Int32); - remaining -= sizeof(Int32); - - if (key < 0 || key > DELETION_VECTOR_MAX_KEY) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap key: {}", key); - if (key <= last_key) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap keys must be sorted in ascending order"); - - auto bitmap = readRoaringPortableSafe(ptr, remaining, key); - - const size_t bitmap_size = bitmap.getSizeInBytes(/*portable=*/true); - if (bitmap_size > remaining) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector roaring bitmap at key {} exceeds blob size", key); - - const UInt64 bitmap_cardinality = bitmap.cardinality(); - UInt64 new_running_cardinality = 0; - if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality exceeds declared cardinality {}", - expected_cardinality); - - if (new_running_cardinality > expected_cardinality) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality {} exceeds declared cardinality {}", - new_running_cardinality, - expected_cardinality); - - running_cardinality = new_running_cardinality; - - for (UInt32 sub_position : bitmap) - { - const UInt64 position = positionFromKeyAndSubPosition(static_cast(key), sub_position); - if (position > static_cast(DELETION_VECTOR_MAX_POSITION)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector position {} is out of supported range", position); - positions.insertValue(position); - } - - ptr += bitmap_size; - remaining -= bitmap_size; - last_key = key; - --remaining_count; - } - - if (remaining != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap has {} trailing bytes", remaining); - - if (running_cardinality != expected_cardinality) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality {} does not match deserialized row count {}", - expected_cardinality, - running_cardinality); -} - -std::string_view extractDeletionVectorPayload(std::string_view blob) -{ - if (blob.size() < 12) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); - - const auto * blob_bytes = reinterpret_cast(blob.data()); - ReadBufferFromMemory mem_buf(blob.data(), blob.size()); - UInt32 combined_length = 0; - readBinaryBigEndian(combined_length, mem_buf); - if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); - - const size_t vector_size = combined_length - sizeof(DELETION_VECTOR_MAGIC); - const size_t expected_blob_size = sizeof(UInt32) + combined_length + sizeof(UInt32); - if (blob.size() != expected_blob_size) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob size {} does not match combined length {}", blob.size(), combined_length); - - if (std::memcmp(blob_bytes + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); - - const UInt8 * crc_input = blob_bytes + sizeof(UInt32); - const size_t crc_input_size = combined_length; - mem_buf.ignore(combined_length); - UInt32 expected_crc = 0; - readBinaryBigEndian(expected_crc, mem_buf); - const UInt32 actual_crc = static_cast(crc32_z(0L, reinterpret_cast(crc_input), crc_input_size)); - if (expected_crc != actual_crc) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector CRC mismatch"); - - return std::string_view(blob.data() + 2 * sizeof(UInt32), vector_size); -} - -void deserializeDeletionVectorV1(std::string_view blob, UInt64 expected_cardinality, ColumnUInt64 & positions) -{ - if (expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality {} exceeds materialization limit {}", - expected_cardinality, - PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); - - deserializeRoaringPositionBitmap(extractDeletionVectorPayload(blob), expected_cardinality, positions); -} - NamesAndTypesList getPuffinMetadataSchema() { return { @@ -815,6 +231,7 @@ Chunk PuffinMetadataInputFormat::read() { blob_index = 0; footer = readPuffinFooter(*in, seekable_read); + /// Metadata never reads blob payloads; drop the full-file buffer from the non-seekable path. footer.data.clear(); footer.data.shrink_to_fit(); initialized = true; @@ -892,6 +309,7 @@ Chunk PuffinInputFormat::read() { blob_index = 0; footer = readPuffinFooter(*in, seekable_read); + /// No deletion-vector payload will be read; drop the full-file buffer from the non-seekable path. if (!need_deleted_rows) { footer.data.clear(); @@ -905,13 +323,8 @@ Chunk PuffinInputFormat::read() const size_t current_blob_index = blob_index; const auto & blob = footer.blobs[blob_index++]; - if (blob.type != PUFFIN_DELETION_VECTOR_BLOB_TYPE) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: unexpected blob type '{}', only '{}' is supported", - current_blob_index, - blob.type, - PUFFIN_DELETION_VECTOR_BLOB_TYPE); + if (blob.type != "deletion-vector-v1") + continue; const auto & referenced_data_file = blob.properties.at("referenced-data-file"); @@ -926,11 +339,24 @@ Chunk PuffinInputFormat::read() col_file->insertData(referenced_data_file.data(), referenced_data_file.size()); MutableColumnPtr col_rows; + // Payload I/O and validation (envelope / CRC / roaring) run only when `deleted_rows` is + // projected. Footer property checks above stay projection-independent. + // Intentional: subset reads of `referenced_data_file` must not force reading up to + // PUFFIN_DV_MAX_BLOB_SIZE for blobs that will never be materialized. + // Materialization ceiling uses footer `cardinality` and must run before blob allocate. if (need_deleted_rows) { + if (expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + const String blob_data = readDeletionVectorBlobBytes(blob, *in, footer.data, seekable_read); + const auto positions = deserializeDeletionVectorV1Blob(blob_data, expected_cardinality); auto col_rows_data = ColumnUInt64::create(); - deserializeDeletionVectorV1(blob_data, expected_cardinality, *col_rows_data); + col_rows_data->getData().insert(positions.begin(), positions.end()); auto col_rows_offsets = ColumnArray::ColumnOffsets::create(); col_rows_offsets->insertValue(col_rows_data->size()); @@ -1004,8 +430,6 @@ void registerInputFormatPuffin(FormatFactory & factory) Special input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) file footer metadata. It outputs one row per blob entry from the footer `BlobMetadata` list. -`deletion-vector-v1` is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. - Fixed output columns: - `blob_type` (`String`) - blob type, for example `deletion-vector-v1` - `snapshot_id` (`Int64`) - snapshot id of the blob @@ -1018,7 +442,7 @@ Fixed output columns: Optional top-level `FileMetadata.properties` in the footer (for example `created-by`) are type-checked when present but are not returned as columns. If the key is present it must be a JSON object with string values (null is rejected). -LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. Non-seekable input (pipes / `input_format_allow_seeks = 0`) buffers the file up to an absolute ceiling sized for one max deletion-vector blob plus max footer; larger inputs require seekable reads. ## Example usage {#example-usage} @@ -1047,18 +471,18 @@ Pair with the `Puffin` format to read `deletion-vector-v1` blob payloads. Input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) files. -The format exposes deleted row positions from `deletion-vector-v1` blobs. It is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. +The format exposes deleted row positions from `deletion-vector-v1` blobs. Other blob types (for example `apache-datasketches-theta-v1`) are skipped. If a puffin file contains multiple `deletion-vector-v1` blobs, the format outputs one row per such blob. Fixed output columns: - `referenced_data_file` (`String`) - location of the data file the deletion vector applies to (`referenced-data-file` blob property) - `deleted_rows` (`Array(UInt64)`) - 64-bit row positions deleted according to the deletion vector roaring bitmap -Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. +Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested, before the blob is allocated. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. -LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. Non-seekable input (pipes / `input_format_allow_seeks = 0`) buffers the file up to an absolute ceiling sized for one max deletion-vector blob plus max footer; larger inputs require seekable reads. Only a subset of output columns can be requested. A user-provided structure with unexpected column names or types is rejected when the format is created. diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h index 203dcd368544..cf2cf5e1e5d3 100644 --- a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h @@ -4,22 +4,11 @@ #include #include +#include namespace DB { -struct PuffinBlob -{ - String type; - Int64 snapshot_id = 0; - Int64 sequence_number = 0; - std::vector fields; - Int64 offset = 0; - Int64 length = 0; - String compression_codec; - std::map properties; -}; - struct PuffinFooter { std::vector blobs; diff --git a/src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp b/src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp new file mode 100644 index 000000000000..d9be92b01737 --- /dev/null +++ b/src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp @@ -0,0 +1,88 @@ +#include + +#include + +#if USE_PARQUET + +#include +#include +#include + +#include + +using namespace DB; +using namespace DB::Parquet; + +namespace DB::ErrorCodes +{ +extern const int INCORRECT_DATA; +} + +namespace +{ + +parq::FileMetaData makeFileMetaData(Int64 file_num_rows, const std::vector & row_group_num_rows) +{ + parq::FileMetaData meta; + meta.__set_num_rows(file_num_rows); + meta.row_groups.reserve(row_group_num_rows.size()); + for (Int64 num_rows : row_group_num_rows) + { + parq::RowGroup row_group; + row_group.__set_num_rows(num_rows); + meta.row_groups.push_back(std::move(row_group)); + } + return meta; +} + +} + +TEST(ParquetRowGroupGlobalOffsets, BuildsPrefixOffsets) +{ + const auto offsets = buildRowGroupGlobalOffsets(makeFileMetaData(30, {10, 0, 20})); + ASSERT_EQ(offsets.size(), 4u); + EXPECT_EQ(offsets[0], 0u); + EXPECT_EQ(offsets[1], 10u); + EXPECT_EQ(offsets[2], 10u); + EXPECT_EQ(offsets[3], 30u); +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsNegativeFileRowCount) +{ + EXPECT_THROW(buildRowGroupGlobalOffsets(makeFileMetaData(-1, {})), Exception); +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsNegativeRowGroupRowCount) +{ + EXPECT_THROW(buildRowGroupGlobalOffsets(makeFileMetaData(10, {10, -1})), Exception); +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsMismatchWithFileNumRows) +{ + try + { + buildRowGroupGlobalOffsets(makeFileMetaData(11, {10, 0})); + FAIL() << "Expected INCORRECT_DATA"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::INCORRECT_DATA); + } +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsOverflowingCumulativeCounts) +{ + constexpr Int64 max_rows = std::numeric_limits::max(); + /// Two Int64::max values fit in UInt64; the third overflows checked addition. + try + { + buildRowGroupGlobalOffsets(makeFileMetaData(max_rows, {max_rows, max_rows, max_rows})); + FAIL() << "Expected INCORRECT_DATA"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::INCORRECT_DATA); + } +} + +#endif diff --git a/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp b/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp new file mode 100644 index 000000000000..80be25af4ce8 --- /dev/null +++ b/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp @@ -0,0 +1,192 @@ +#include + +#include + +#if USE_PARQUET + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int LOGICAL_ERROR; +} + +namespace +{ + +void writeMultiRowGroupParquet(const String & path, size_t rows_per_group, size_t num_groups) +{ + FormatSettings format_settings; + format_settings.parquet.row_group_rows = rows_per_group; + format_settings.parquet.parallel_encoding = false; + + Block header; + header.insert(ColumnWithTypeAndName(std::make_shared(), "x")); + + Chunks chunks; + for (size_t group = 0; group < num_groups; ++group) + { + auto column = ColumnUInt64::create(); + for (size_t i = 0; i < rows_per_group; ++i) + column->insert(group * rows_per_group + i); + chunks.emplace_back(Columns{std::move(column)}, rows_per_group); + } + + auto source = std::make_shared(std::make_shared(header), std::move(chunks)); + QueryPipelineBuilder pipeline_builder; + pipeline_builder.init(Pipe(source)); + auto pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline_builder)); + + WriteBufferFromFile write_buffer(path); + auto output = std::make_shared(write_buffer, pipeline.getSharedHeader(), format_settings, nullptr); + pipeline.complete(output); + CompletedPipelineExecutor executor(pipeline); + executor.execute(); + output->finalize(); + write_buffer.finalize(); +} + +size_t readNeedOnlyCountTotal( + ReadBuffer & in, + const SharedHeader & header, + const FormatSettings & format_settings, + FormatParserSharedResourcesPtr parser_shared_resources, + const FileBucketInfoPtr & buckets) +{ + auto input = std::make_shared( + in, + header, + format_settings, + parser_shared_resources, + std::make_shared(), + /*min_bytes_for_seek=*/ 1024); + if (buckets) + input->setBucketsToRead(buckets); + input->needOnlyCount(); + + size_t total = 0; + while (true) + { + Chunk chunk = input->generate(); + if (!chunk) + break; + total += chunk.getNumRows(); + } + return total; +} + +std::vector> readNeedOnlyCountOffsetsAndRows( + ReadBuffer & in, + const SharedHeader & header, + const FormatSettings & format_settings, + FormatParserSharedResourcesPtr parser_shared_resources, + const FileBucketInfoPtr & buckets) +{ + auto input = std::make_shared( + in, + header, + format_settings, + parser_shared_resources, + std::make_shared(), + /*min_bytes_for_seek=*/ 1024); + if (buckets) + input->setBucketsToRead(buckets); + input->needOnlyCount(); + + std::vector> result; + while (true) + { + Chunk chunk = input->generate(); + if (!chunk) + break; + auto info = chunk.getChunkInfos().get(); + if (!info) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ChunkInfoRowNumbers on need_only_count chunk"); + result.emplace_back(info->row_num_offset, chunk.getNumRows()); + } + return result; +} + +} + +TEST(ParquetV3NeedOnlyCountBuckets, CountsOnlyAssignedRowGroups) +{ + tryRegisterFormats(); + const auto context = getContext().context; + + Poco::TemporaryFile temp_file; + const String path = temp_file.path(); + constexpr size_t rows_per_group = 10; + constexpr size_t num_groups = 3; + writeMultiRowGroupParquet(path, rows_per_group, num_groups); + + Block header; + header.insert(ColumnWithTypeAndName(std::make_shared(), "x")); + auto shared_header = std::make_shared(header); + FormatSettings format_settings; + auto parser_shared_resources = FormatParserSharedResources::singleThreaded(context->getSettingsRef()); + + { + ReadBufferFromFile in(path); + EXPECT_EQ( + readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, /*buckets=*/ nullptr), + rows_per_group * num_groups); + } + + { + /// Unbucketed need-only-count emits one span per row group (not a single whole-file chunk). + ReadBufferFromFile in(path); + const auto offsets_and_rows + = readNeedOnlyCountOffsetsAndRows(in, shared_header, format_settings, parser_shared_resources, /*buckets=*/ nullptr); + ASSERT_EQ(offsets_and_rows.size(), num_groups); + for (size_t group = 0; group < num_groups; ++group) + { + EXPECT_EQ(offsets_and_rows[group].first, group * rows_per_group); + EXPECT_EQ(offsets_and_rows[group].second, rows_per_group); + } + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{1}); + EXPECT_EQ(readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, buckets), rows_per_group); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{0, 2}); + const auto offsets_and_rows + = readNeedOnlyCountOffsetsAndRows(in, shared_header, format_settings, parser_shared_resources, buckets); + ASSERT_EQ(offsets_and_rows.size(), 2u); + EXPECT_EQ(offsets_and_rows[0].first, 0u); + EXPECT_EQ(offsets_and_rows[0].second, rows_per_group); + EXPECT_EQ(offsets_and_rows[1].first, 2 * rows_per_group); + EXPECT_EQ(offsets_and_rows[1].second, rows_per_group); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{}); + EXPECT_EQ(readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, buckets), 0u); + } +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index 4517834d0dec..75861568b6d0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include namespace DB::ErrorCodes { @@ -242,6 +244,22 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE const auto record_count = getValueFromRowByName(row_index, c_data_file_record_count, TypeIndex::Int64).safeGet(); const auto file_size_in_bytes = getValueFromRowByName(row_index, c_data_file_file_size_in_bytes, TypeIndex::Int64).safeGet(); + std::optional content_offset; + if (hasPath(c_data_file_content_offset)) + { + const auto content_offset_value = getValueFromRowByName(row_index, c_data_file_content_offset); + if (!content_offset_value.isNull()) + content_offset = content_offset_value.safeGet(); + } + + std::optional content_size_in_bytes; + if (hasPath(c_data_file_content_size_in_bytes)) + { + const auto content_size_value = getValueFromRowByName(row_index, c_data_file_content_size_in_bytes); + if (!content_size_value.isNull()) + content_size_in_bytes = content_size_value.safeGet(); + } + switch (content_type) { case FileContentType::DATA: { @@ -265,6 +283,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE } case FileContentType::POSITION_DELETE: { /// reference_file_path can be absent in schema for some reason, though it is present in specification: https://iceberg.apache.org/spec/#manifests + const bool is_puffin = Poco::toLower(file_format) == "puffin"; std::optional lower_reference_data_file_path; std::optional upper_reference_data_file_path; bool bounds_set_by_referenced_data_file = false; @@ -280,7 +299,9 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE bounds_set_by_referenced_data_file = true; } } - if (!bounds_set_by_referenced_data_file) + /// Parquet position deletes may fall back to file-path column bounds. Puffin deletion + /// vectors must use the dedicated referenced_data_file field only. + if (!bounds_set_by_referenced_data_file && !is_puffin) { if (auto it = value_for_bounds.find(IcebergPositionDeleteTransform::data_file_path_column_field_id); it != value_for_bounds.end()) @@ -292,6 +313,20 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE upper_reference_data_file_path.emplace(Iceberg::IcebergPathFromMetadata::deserialize(upper.safeGet())); } } + + if (is_puffin) + { + if (!content_offset.has_value() || !content_size_in_bytes.has_value()) + { + throw Exception( + DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Puffin deletion vector entry in manifest file '{}' is missing content_offset or content_size_in_bytes", + manifest_file_path); + } + requireDirectReferencedDataFileForPuffinDeletionVector( + bounds_set_by_referenced_data_file, lower_reference_data_file_path, manifest_file_path); + } + return std::make_shared( FileContentType::POSITION_DELETE, file_path_key, @@ -308,7 +343,9 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE /*equality_ids*/ std::nullopt, /*sort_order_id = */ std::nullopt, record_count, - file_size_in_bytes); + file_size_in_bytes, + content_offset, + content_size_in_bytes); } case FileContentType::EQUALITY_DELETE: { std::vector equality_ids; diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h b/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h index fcc2c0495e18..566b1965e0df 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h @@ -1,6 +1,8 @@ #pragma once #include +#include + namespace DB { template @@ -17,4 +19,11 @@ struct DataLakeObjectMetadata ExcludedRowsPtr excluded_rows; }; +/// True when a deletion vector (or similar) will filter rows via DeletionVectorTransform. +/// Count-from-files cache must skip these objects: the cache key is data-file identity only, +/// while excluded_rows can change independently (Iceberg puffin DVs, DeltaLake selection vectors). +/// Cluster task serialization must also fail closed when these cannot be carried on the wire. +bool hasNonEmptyExcludedRows(const DataLakeObjectMetadata & metadata); +bool hasNonEmptyExcludedRows(const std::optional & metadata); + } diff --git a/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp b/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp index 3fd353ef99a1..abe97a78d149 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB::ErrorCodes @@ -12,6 +13,69 @@ namespace DB::ErrorCodes namespace DB { +namespace +{ + +bool isConstCountOnlyChunk(const Chunk & chunk, const ChunkInfoRowNumbers & chunk_info) +{ + /// need_only_count emits ColumnConst defaults with no prior filter. Dense filtering is still + /// required when real column values or an applied_filter (PREWHERE / prior filters) are present. + if (chunk_info.applied_filter.has_value()) + return false; + + for (const auto & column : chunk.getColumns()) + { + if (!isColumnConst(*column)) + return false; + } + return true; +} + +void transformConstCountOnlyChunk(Chunk & chunk, ChunkInfoRowNumbers & chunk_info, const DataLakeObjectMetadata::ExcludedRows & excluded_rows) +{ + const size_t num_rows_before = chunk.getNumRows(); + size_t range_end = 0; + if (common::addOverflow(chunk_info.row_num_offset, num_rows_before, range_end)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Deletion vector count range overflows size_t: offset {} num_rows {}", + chunk_info.row_num_offset, + num_rows_before); + } + + const UInt64 deleted = excluded_rows.rb_range_cardinality(chunk_info.row_num_offset, range_end); + if (deleted > num_rows_before) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Deletion vector reports {} deletes in a count chunk of {} rows (offset {})", + deleted, + num_rows_before, + chunk_info.row_num_offset); + } + + const size_t num_rows_after = num_rows_before - static_cast(deleted); + if (num_rows_after == num_rows_before) + return; + + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->cloneResized(num_rows_after); + chunk.setColumns(std::move(columns), num_rows_after); +} + +} + +bool hasNonEmptyExcludedRows(const DataLakeObjectMetadata & metadata) +{ + return metadata.excluded_rows && metadata.excluded_rows->size() > 0; +} + +bool hasNonEmptyExcludedRows(const std::optional & metadata) +{ + return metadata.has_value() && hasNonEmptyExcludedRows(*metadata); +} DeletionVectorTransform::DeletionVectorTransform( const DB::SharedHeader & header_, @@ -32,6 +96,12 @@ void DeletionVectorTransform::transform(DB::Chunk & chunk, const ExcludedRows & if (!chunk_info) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "ChunkInfoRowNumbers does not exist"); + if (isConstCountOnlyChunk(chunk, *chunk_info)) + { + transformConstCountOnlyChunk(chunk, *chunk_info, excluded_rows); + return; + } + const size_t num_rows_before = chunk.getNumRows(); size_t num_rows_after = num_rows_before; size_t idx_in_chunk = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h index b87ff718e172..d175864e23af 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h @@ -176,6 +176,8 @@ DEFINE_ICEBERG_FIELD_COMPOUND(data_file, null_value_counts); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, lower_bounds); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, upper_bounds); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, referenced_data_file); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, content_offset); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, content_size_in_bytes); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, sort_order_id); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, record_count); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, file_size_in_bytes); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index 4e161ca8c863..6e99248bcba4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace DB::ErrorCodes { @@ -25,6 +26,23 @@ extern const int UNKNOWN_PROTOCOL; using namespace DB::Iceberg; +namespace DB::Iceberg +{ + +void requireParquetDataFileForRowDeletes(const String & file_format, std::string_view feature_name) +{ + if (Poco::toUpper(file_format) != "PARQUET") + { + throw Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "{} are only supported for data files of Parquet format in Iceberg, but got {}", + feature_name, + file_format); + } +} + +} + namespace DB { @@ -97,13 +115,7 @@ std::shared_ptr IcebergDataObjectInfo::getPositionDeleteTransf void IcebergDataObjectInfo::addPositionDeleteObject(Iceberg::ProcessedManifestFileEntryPtr position_delete_object, const String & resolved_storage_path) { - if (Poco::toUpper(info.file_format) != "PARQUET") - { - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Position deletes are only supported for data files of Parquet format in Iceberg, but got {}", - info.file_format); - } + Iceberg::requireParquetDataFileForRowDeletes(info.file_format, "Position deletes"); info.position_deletes_objects.emplace_back( resolved_storage_path, position_delete_object->parsed_entry->file_format, std::nullopt, position_delete_object->sequence_number); @@ -118,6 +130,26 @@ void IcebergDataObjectInfo::addEqualityDeleteObject(const Iceberg::ProcessedMani equality_delete_object->resolved_schema_id); } +ObjectInfoPtr IcebergDataObjectInfo::clone() const +{ + auto result = std::make_shared(relative_path_with_metadata, info); + result->data_lake_metadata = data_lake_metadata; + result->file_bucket_info = file_bucket_info; + return result; +} + +bool hasIcebergEqualityDeletes(const ObjectInfoPtr & object_info) +{ + const auto * iceberg = dynamic_cast(object_info.get()); + return iceberg && !iceberg->info.equality_deletes_objects.empty(); +} + +bool hasIcebergPositionDeletes(const ObjectInfoPtr & object_info) +{ + const auto * iceberg = dynamic_cast(object_info.get()); + return iceberg && !iceberg->info.position_deletes_objects.empty(); +} + #endif void IcebergObjectSerializableInfo::serializeForClusterFunctionProtocol(WriteBuffer & out, size_t protocol_version) const diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h index 6763d1013201..9e646eedd52c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h @@ -11,12 +11,18 @@ #include #include +#include + namespace DB::Iceberg { String computePartitionId(const Row & partition_key_value); +/// Position deletes and deletion vectors need file-relative row numbers (`ChunkInfoRowNumbers`), +/// which Parquet readers emit. Reject other data-file formats early with NOT_IMPLEMENTED. +void requireParquetDataFileForRowDeletes(const String & file_format, std::string_view feature_name); + struct IcebergObjectSerializableInfo { @@ -84,10 +90,18 @@ struct IcebergDataObjectInfo : public ObjectInfo, std::enable_shared_from_this; + +/// Equality deletes need real column values — disable need_only_count / count-from-files cache. +bool hasIcebergEqualityDeletes(const ObjectInfoPtr & object_info); +/// Position-delete attachment changes independently of data-file identity — skip count-from-files cache. +bool hasIcebergPositionDeletes(const ObjectInfoPtr & object_info); } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp new file mode 100644 index 000000000000..f7d815c05264 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -0,0 +1,273 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ICEBERG_SPECIFICATION_VIOLATION; + extern const int BAD_ARGUMENTS; +} + +namespace Setting +{ +extern const SettingsBool use_puffin_files_cache; +} + +} + +namespace DB::Iceberg +{ + +void validateDeletionVectorPositionsAgainstDataFile( + std::span deleted_positions, + UInt64 expected_cardinality, + Int64 data_file_record_count) +{ + if (data_file_record_count < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Data file record_count {} must be non-negative", + data_file_record_count); + } + + const UInt64 data_file_rows = static_cast(data_file_record_count); + + if (expected_cardinality > data_file_rows) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector cardinality {} exceeds data file record_count {}", + expected_cardinality, + data_file_record_count); + } + + for (UInt64 position : deleted_positions) + { + if (position >= data_file_rows) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector position {} is out of range for data file record_count {}", + position, + data_file_record_count); + } + } +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVectorUncached( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + UInt64 expected_cardinality, + Int64 data_file_record_count, + ContextPtr context, + LoggerPtr log, + bool disable_filesystem_cache) +{ + RelativePathWithMetadata puffin_object{puffin_path}; + auto read_settings = context->getReadSettings(); + if (disable_filesystem_cache) + read_settings.enable_filesystem_cache = false; + + auto read_buffer = createReadBuffer(puffin_object, object_storage, context, log, read_settings); + + auto * seekable = dynamic_cast(read_buffer.get()); + if (!seekable) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin deletion vector read requires a seekable buffer"); + + auto file_size = tryGetFileSizeFromReadBuffer(*read_buffer); + if (!file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot determine Puffin file size for '{}'", puffin_path); + + const auto blobs = readPuffinFooterBlobsFromSeekable(*seekable, *file_size); + bindDeletionVectorBlob( + blobs, + content_offset, + content_size_in_bytes, + expected_data_file.serialize(), + expected_cardinality); + + auto deleted_positions = readDeletionVectorFromPuffin( + *read_buffer, content_offset, content_size_in_bytes, expected_cardinality); + + validateDeletionVectorPositionsAgainstDataFile(deleted_positions, expected_cardinality, data_file_record_count); + + if (deleted_positions.empty()) + return nullptr; + + auto bitmap = std::make_shared(); + for (UInt64 position : deleted_positions) + bitmap->add(static_cast(position)); + + LOG_DEBUG( + log, + "Loaded deletion vector from puffin file '{}' for data file '{}': {} deleted rows", + puffin_path, + expected_data_file.serialize(), + deleted_positions.size()); + + return bitmap; +} + +} + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + const std::optional & referenced_data_file, + Int64 expected_cardinality, + Int64 data_file_record_count, + ContextPtr context, + LoggerPtr log) +{ + if (referenced_data_file.has_value() && referenced_data_file.value() != expected_data_file) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector referenced_data_file '{}' does not match data file '{}'", + referenced_data_file->serialize(), + expected_data_file.serialize()); + } + + if (expected_cardinality < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector record_count {} must be non-negative", + expected_cardinality); + } + + if (data_file_record_count < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Data file record_count {} must be non-negative", + data_file_record_count); + } + + const UInt64 expected_cardinality_u64 = static_cast(expected_cardinality); + + /// Fail closed before I/O when declared DV cardinality cannot fit in the data file. + if (expected_cardinality_u64 > static_cast(data_file_record_count)) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector cardinality {} exceeds data file record_count {}", + expected_cardinality, + data_file_record_count); + } + + const bool use_cache = context->getSettingsRef()[Setting::use_puffin_files_cache]; + if (!use_cache) + { + LOG_TRACE(log, "Not using Puffin files cache for '{}', because the setting use_puffin_files_cache is false", puffin_path); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false); + } + + RelativePathWithMetadata puffin_object{puffin_path}; + if (!puffin_object.metadata) + puffin_object.metadata = object_storage->getObjectMetadata(puffin_object.getPath(), /*with_tags=*/ false); + + if (!puffin_object.metadata->isEtagUsableAsCacheKey()) + { + LOG_TRACE( + log, + "Not using Puffin files cache for '{}', because etag is empty or not a strong content identifier", + puffin_path); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false); + } + + auto cache_key = PuffinFilesCache::tryCreateKey( + PuffinFilesCache::makeStorageIdentity(*object_storage), + puffin_path, + puffin_object.metadata->etag, + content_offset, + content_size_in_bytes, + expected_data_file.serialize(), + expected_cardinality_u64, + static_cast(data_file_record_count)); + + if (!cache_key) + { + LOG_TRACE(log, "Not using Puffin files cache for '{}', because etag is empty", puffin_path); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false); + } + + auto cache = context->getPuffinFilesCache(); + return cache->getOrSetDeletionVector(*cache_key, [&]() + { + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + true); + }); +} + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h new file mode 100644 index 000000000000..a0323730564d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h @@ -0,0 +1,38 @@ +#pragma once +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include + +#include +#include + +namespace DB::Iceberg +{ + +/// Reject DV positions outside `[0, data_file_record_count)` and cardinality that cannot fit. +void validateDeletionVectorPositionsAgainstDataFile( + std::span deleted_positions, + UInt64 expected_cardinality, + Int64 data_file_record_count); + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + const std::optional & referenced_data_file, + Int64 expected_cardinality, + Int64 data_file_record_count, + ContextPtr context, + LoggerPtr log); + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 39cc26a5fd6a..b498be8a92b9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -71,6 +72,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; +extern const int ICEBERG_SPECIFICATION_VIOLATION; } namespace Setting { @@ -270,6 +272,7 @@ IcebergIterator::IcebergIterator( persistent_components_) , blocking_queue(100) , callback(std::move(callback_)) + , local_context(local_context_) { auto delete_file = deletes_iterator.next(); while (delete_file.has_value()) @@ -278,15 +281,25 @@ IcebergIterator::IcebergIterator( { equality_deletes_files.emplace_back(std::move(delete_file.value())); } + else if (delete_file.value()->parsed_entry->isDeletionVector()) + { + deletion_vector_files.emplace_back(std::move(delete_file.value())); + } else { - position_deletes_files.emplace_back(std::move(delete_file.value())); + parquet_position_deletes_files.emplace_back(std::move(delete_file.value())); } delete_file = deletes_iterator.next(); } - LOG_DEBUG(logger, "Taken {} position deletes file and {} equality deletes files in iceberg iterator", position_deletes_files.size(), equality_deletes_files.size()); + LOG_DEBUG( + logger, + "Taken {} deletion vector files, {} parquet position delete files and {} equality delete files in iceberg iterator", + deletion_vector_files.size(), + parquet_position_deletes_files.size(), + equality_deletes_files.size()); std::sort(equality_deletes_files.begin(), equality_deletes_files.end()); - std::sort(position_deletes_files.begin(), position_deletes_files.end()); + std::sort(deletion_vector_files.begin(), deletion_vector_files.end()); + std::sort(parquet_position_deletes_files.begin(), parquet_position_deletes_files.end()); producer_task = std::make_unique( [this, thread_group = CurrentThread::getGroup()]() { @@ -333,42 +346,97 @@ ObjectInfoPtr IcebergIterator::next(size_t) manifest_file_entry, persistent_components.path_resolver.resolve(manifest_file_entry->parsed_entry->file_path_key), table_state_snapshot->schema_id); - for (const auto & position_delete : - defineDeletesSpan(manifest_file_entry, position_deletes_files, /* is_equality_delete */ false, logger)) + + const auto & data_file_path = object_info->info.data_object_file_path_key; + bool has_deletion_vector = false; + + for (const auto & deletion_vector : + defineDeletesSpan(manifest_file_entry, deletion_vector_files, /* is_equality_delete */ false, logger)) { - const auto & data_file_path = object_info->info.data_object_file_path_key; - const auto & lower = position_delete->parsed_entry->lower_reference_data_file_path; - const auto & upper = position_delete->parsed_entry->upper_reference_data_file_path; - bool can_contain_data_file_deletes - = (!lower.has_value() || *lower <= data_file_path) - && (!upper.has_value() || *upper >= data_file_path); - /// Skip position deletes that do not match the data file path. - if (!can_contain_data_file_deletes) + const auto & referenced_data_file = deletion_vector->parsed_entry->lower_reference_data_file_path; + if (!referenced_data_file.has_value() || referenced_data_file.value() != data_file_path) + continue; + + if (has_deletion_vector) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Multiple deletion vectors match data file '{}'", + data_file_path); + } + + Iceberg::requireParquetDataFileForRowDeletes(object_info->info.file_format, "Deletion vectors"); + + if (!object_info->info.record_count.has_value()) { - ProfileEvents::increment(ProfileEvents::IcebergMinMaxPrunedDeleteFiles); - LOG_TEST( - logger, - "Skipping position delete file `{}` for data file `{}` because position delete has out of bounds reference data file " - "bounds: " - "(lower bound: `{}`, upper bound: `{}`)", - position_delete->parsed_entry->file_path_key, - data_file_path, - lower.has_value() ? lower->serialize() : "[no lower bound]", - upper.has_value() ? upper->serialize() : "[no upper bound]"); + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Data file '{}' is missing record_count required to validate deletion vector positions", + data_file_path); } - else + + const auto & parsed_entry = deletion_vector->parsed_entry; + auto excluded_rows = Iceberg::loadDeletionVector( + object_storage, + persistent_components.path_resolver.resolve(parsed_entry->file_path_key), + parsed_entry->content_offset.value(), + parsed_entry->content_size_in_bytes.value(), + data_file_path, + referenced_data_file, + parsed_entry->record_count, + *object_info->info.record_count, + local_context, + logger); + + object_info->data_lake_metadata.emplace(); + if (excluded_rows) + object_info->data_lake_metadata->excluded_rows = std::move(excluded_rows); + has_deletion_vector = true; + LOG_DEBUG( + logger, + "Attached deletion vector from puffin file `{}` to data file `{}`", + parsed_entry->file_path_key, + data_file_path); + } + + if (!has_deletion_vector) + { + for (const auto & position_delete : + defineDeletesSpan(manifest_file_entry, parquet_position_deletes_files, /* is_equality_delete */ false, logger)) { - ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); - LOG_TEST( - logger, - "Processing position delete file `{}` for data file `{}` with reference data file bounds: " - "(lower bound: `{}`, upper bound: `{}`)", - position_delete->parsed_entry->file_path_key, - data_file_path, - lower.has_value() ? lower->serialize() : "[no lower bound]", - upper.has_value() ? upper->serialize() : "[no upper bound]"); - object_info->addPositionDeleteObject( - position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); + const auto & lower = position_delete->parsed_entry->lower_reference_data_file_path; + const auto & upper = position_delete->parsed_entry->upper_reference_data_file_path; + bool can_contain_data_file_deletes + = (!lower.has_value() || *lower <= data_file_path) + && (!upper.has_value() || *upper >= data_file_path); + /// Skip position deletes that do not match the data file path. + if (!can_contain_data_file_deletes) + { + ProfileEvents::increment(ProfileEvents::IcebergMinMaxPrunedDeleteFiles); + LOG_TEST( + logger, + "Skipping position delete file `{}` for data file `{}` because position delete has out of bounds reference data file " + "bounds: " + "(lower bound: `{}`, upper bound: `{}`)", + position_delete->parsed_entry->file_path_key, + data_file_path, + lower.has_value() ? lower->serialize() : "[no lower bound]", + upper.has_value() ? upper->serialize() : "[no upper bound]"); + } + else + { + ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); + LOG_TEST( + logger, + "Processing position delete file `{}` for data file `{}` with reference data file bounds: " + "(lower bound: `{}`, upper bound: `{}`)", + position_delete->parsed_entry->file_path_key, + data_file_path, + lower.has_value() ? lower->serialize() : "[no lower bound]", + upper.has_value() ? upper->serialize() : "[no upper bound]"); + object_info->addPositionDeleteObject( + position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); + } } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index 09c21b8ec292..973c40440470 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -93,8 +93,10 @@ class IcebergIterator : public IObjectIterator ConcurrentBoundedQueue blocking_queue; std::unique_ptr producer_task; IDataLakeMetadata::FileProgressCallback callback; - std::vector position_deletes_files; + std::vector deletion_vector_files; + std::vector parquet_position_deletes_files; std::vector equality_deletes_files; + ContextPtr local_context; std::exception_ptr exception; std::mutex exception_mutex; }; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index a7bd62ccd110..d81124e47572 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -393,6 +393,7 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO std::optional total_rows; std::optional total_bytes; std::optional total_position_deletes; + std::optional total_equality_deletes; if (snapshot_object->has(f_summary)) { @@ -407,6 +408,9 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO { total_position_deletes = summary_object->getValue(f_total_position_deletes); } + + if (summary_object->has(f_total_equality_deletes)) + total_equality_deletes = summary_object->getValue(f_total_equality_deletes); } if (!snapshot_object->has(f_schema_id)) @@ -420,7 +424,8 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO schema_id, total_rows, total_bytes, - total_position_deletes); + total_position_deletes, + total_equality_deletes); } IcebergDataSnapshotPtr @@ -1048,30 +1053,62 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const return 0; } - - /// All these "hints" with total rows or bytes are optional both in - /// metadata files and in manifest files, so we try all of them one by one - if (auto total_rows = actual_data_snapshot->getTotalRows(); total_rows.has_value()) - { - ProfileEvents::increment(ProfileEvents::IcebergTrivialCountOptimizationApplied); - return total_rows; - } - - Int64 result = 0; + /// Equality deletes remove data rows by value match; summary `total-equality-deletes` counts + /// rows in delete files, not deleted data rows. Fail closed when the field is present and > 0. + if (actual_data_snapshot->total_equality_delete_rows.has_value() + && *actual_data_snapshot->total_equality_delete_rows > 0) + return {}; + + /// Row counts stored in the metadata layers above the manifest files are not used as + /// data sources, because writers derive them instead of measuring them against the data: + /// - the snapshot summary's `total-records` / `total-position-deletes` are maintained + /// incrementally (parent total plus this commit's delta), so a single corrupted commit + /// anywhere in the table history silently poisons every later snapshot -- observed in + /// the wild, making SELECT count() disagree with a full scan of the very same table; + /// - the manifest-list per-entry `added_rows_count`/`existing_rows_count` are stamped + /// from snapshot summary fields by some writers (ClickHouse itself among them): a + /// rewritten manifest list can list every manifest with `added_rows_count = 0` taken + /// from a compaction snapshot's `added-records = 0`, so trusting these counts turned + /// count() into 0 on a perfectly healthy table; + /// - subtracting live position-delete / deletion-vector `record_count` from data-file + /// totals is also unsafe: position delete records may be duplicated across delete + /// files, may reference data files no longer in the snapshot, and Iceberg readers + /// ignore matching parquet position deletes when a DV applies (so raw subtraction + /// double-counts). + /// The manifest files are the ground truth: the per-data-file `record_count` is a + /// required field in every format version, so summing it over the live data files is + /// exact when no live delete files exist. Any live equality / position deletes (including + /// puffin deletion vectors) fail closed to a real scan. + UInt64 result = 0; for (const auto & manifest_list_entry : actual_data_snapshot->manifest_list_entries) { auto manifest_file_ptr = getManifestFileEntriesHandle( object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id); - auto data_count = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); - auto position_deletes_count = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::POSITION_DELETE); - if (!data_count.has_value() || !position_deletes_count.has_value()) + + if (!manifest_file_ptr.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE).empty() + || !manifest_file_ptr.getFilesWithoutDeleted(FileContentType::POSITION_DELETE).empty()) return {}; - result += data_count.value() - position_deletes_count.value(); + /// nullopt means a corrupted manifest file with a negative `record_count`: fail + /// closed to a real scan instead of returning a wrong count. + auto manifest_rows = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); + if (!manifest_rows.has_value()) + return {}; + result += *manifest_rows; } + if (actual_data_snapshot->total_rows.has_value() && *actual_data_snapshot->total_rows != result) + LOG_WARNING( + log, + "Iceberg snapshot summary of table {} claims {} total rows, but its manifest files describe {} rows. " + "The snapshot summary is inconsistent with the table data (possibly a corrupted commit in the table " + "history), using the row count from the manifest files", + persistent_components.table_location, + *actual_data_snapshot->total_rows, + result); + ProfileEvents::increment(ProfileEvents::IcebergTrivialCountOptimizationApplied); - return result; + return static_cast(result); } std::optional IcebergMetadata::totalBytes(ContextPtr local_context) const diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp index f289f3d80d40..b8b239408175 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp @@ -13,6 +13,7 @@ namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; + extern const int ICEBERG_SPECIFICATION_VIOLATION; } namespace DB::Iceberg @@ -32,6 +33,37 @@ String FileContentTypeToString(FileContentType type) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unsupported content type: {}", static_cast(type)); } +PositionDeleteKindPresence getPositionDeleteKindPresence( + const std::vector & position_delete_files) +{ + PositionDeleteKindPresence presence; + for (const auto & file : position_delete_files) + { + if (file->parsed_entry->isDeletionVector()) + presence.has_deletion_vectors = true; + else + presence.has_parquet_position_deletes = true; + + if (presence.hasBoth()) + break; + } + return presence; +} + +void requireDirectReferencedDataFileForPuffinDeletionVector( + bool set_from_referenced_data_file_field, + const std::optional & referenced_path, + const IcebergPathFromMetadata & manifest_file_path) +{ + if (!set_from_referenced_data_file_field || !referenced_path.has_value() || referenced_path->empty()) + { + throw DB::Exception( + DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Puffin deletion vector entry in manifest file '{}' is missing referenced_data_file", + manifest_file_path); + } +} + static std::strong_ordering operator<=>(const PartitionSpecsEntry & lhs, const PartitionSpecsEntry & rhs) { return std::tie(lhs.source_id, lhs.transform_name, lhs.partition_name) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index f7c2ced00bf3..41de844cef25 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -15,6 +15,7 @@ #include #include +#include namespace DB::Iceberg { @@ -94,6 +95,17 @@ struct ParsedManifestFileEntry : boost::noncopyable Int64 record_count; Int64 file_size_in_bytes; + /// Iceberg v3 deletion vector metadata (position delete entries with puffin format) + std::optional content_offset; + std::optional content_size_in_bytes; + + bool isDeletionVector() const + { + return Poco::toLower(file_format) == "puffin" + && content_offset.has_value() + && content_size_in_bytes.has_value(); + } + ParsedManifestFileEntry( FileContentType content_type_, IcebergPathFromMetadata file_path_key_, @@ -110,7 +122,9 @@ struct ParsedManifestFileEntry : boost::noncopyable std::optional> equality_ids_, std::optional sort_order_id_, Int64 record_count_, - Int64 file_size_in_bytes_) + Int64 file_size_in_bytes_, + std::optional content_offset_ = std::nullopt, + std::optional content_size_in_bytes_ = std::nullopt) : content_type(content_type_) , file_path_key(std::move(file_path_key_)) , row_number(row_number_) @@ -127,6 +141,8 @@ struct ParsedManifestFileEntry : boost::noncopyable , sort_order_id(sort_order_id_) , record_count(record_count_) , file_size_in_bytes(file_size_in_bytes_) + , content_offset(content_offset_) + , content_size_in_bytes(content_size_in_bytes_) { } }; @@ -146,6 +162,28 @@ struct ProcessedManifestFileEntry using ProcessedManifestFileEntryPtr = std::shared_ptr; +/// Live POSITION_DELETE entries are either puffin deletion vectors or parquet position-delete files. +/// Iceberg readers ignore matching parquet position deletes when a DV applies. Helpers classify +/// coexistence for callers that need the distinction; IcebergMetadata::totalRows fail-closes on +/// any live position deletes (DV or parquet) rather than subtracting either kind. +struct PositionDeleteKindPresence +{ + bool has_deletion_vectors = false; + bool has_parquet_position_deletes = false; + + bool hasBoth() const { return has_deletion_vectors && has_parquet_position_deletes; } +}; + +PositionDeleteKindPresence getPositionDeleteKindPresence( + const std::vector & position_delete_files); + +/// Puffin deletion vectors must identify the data file via the dedicated `referenced_data_file` +/// manifest field (non-empty). Position-delete lower/upper bounds must not be used as a fallback. +void requireDirectReferencedDataFileForPuffinDeletionVector( + bool set_from_referenced_data_file_field, + const std::optional & referenced_path, + const IcebergPathFromMetadata & manifest_file_path); + bool operator<(const PartitionSpecification & lhs, const PartitionSpecification & rhs); bool operator<(const DB::Row & lhs, const DB::Row & rhs); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index d21cf0de238c..f3d8d378cdcb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -204,6 +204,11 @@ std::optional ManifestFileIterator::ManifestFileEntriesHandle::getBytesCo return result; } +PositionDeleteKindPresence ManifestFileIterator::ManifestFileEntriesHandle::getPositionDeleteKindPresence() const +{ + return Iceberg::getPositionDeleteKindPresence(getFilesWithoutDeleted(FileContentType::POSITION_DELETE)); +} + ManifestFileIterator::ManifestFileEntriesHandle ManifestFileIterator::getFilesWithoutDeletedHandle() const { if (!isInitialized()) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index ac98cc89de95..7785e5178f73 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -60,6 +60,8 @@ class ManifestFileIterator : public boost::noncopyable std::optional getBytesCountInAllDataFilesExcludingDeleted() const; + PositionDeleteKindPresence getPositionDeleteKindPresence() const; + private: friend class ManifestFileIterator; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h index fd46422f60ad..3fbd4aa494d4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h @@ -16,18 +16,18 @@ struct IcebergDataSnapshot DB::ManifestFileCacheKeys manifest_list_entries; Int64 snapshot_id; Int64 schema_id_on_snapshot_commit; + /// Row-count hint from the snapshot summary (`total-records`). Only used to log a + /// warning when it disagrees with the row count derived from the manifest files; never + /// used as a data source, because the summary is maintained incrementally by writers + /// and a corrupted commit in the table history poisons it silently. std::optional total_rows; std::optional total_bytes; std::optional total_position_delete_rows; + /// Rows in equality-delete files (snapshot summary). Not a count of deleted data rows; + /// used only to fail closed trivial COUNT when equality deletes are present. + std::optional total_equality_delete_rows; std::optional partition_key; std::optional sorting_key; - - std::optional getTotalRows() const - { - if (total_rows.has_value() && total_position_delete_rows.has_value()) - return *total_rows - *total_position_delete_rows; - return std::nullopt; - } }; using IcebergDataSnapshotPtr = std::shared_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp new file mode 100644 index 000000000000..f799e50a11b8 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp @@ -0,0 +1,62 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include + +using namespace DB; + +TEST(IcebergCountShortcuts, HasEqualityAndPositionDeleteHelpers) +{ + Iceberg::IcebergObjectSerializableInfo info; + info.data_object_file_path_key = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"); + info.file_format = "PARQUET"; + + auto plain = std::make_shared(RelativePathWithMetadata{"data/file.parquet"}); + EXPECT_FALSE(hasIcebergEqualityDeletes(plain)); + EXPECT_FALSE(hasIcebergPositionDeletes(plain)); + + auto iceberg = std::make_shared(RelativePathWithMetadata{"data/file.parquet"}, info); + EXPECT_FALSE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_FALSE(hasIcebergPositionDeletes(iceberg)); + + iceberg->info.equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "s3://bucket/deletes/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1}, + .schema_id = 0, + }); + EXPECT_TRUE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_FALSE(hasIcebergPositionDeletes(iceberg)); + + iceberg->info.position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "s3://bucket/deletes/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 1, + }); + EXPECT_TRUE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_TRUE(hasIcebergPositionDeletes(iceberg)); +} + +TEST(IcebergCountShortcuts, SnapshotSummaryFieldsRemainLoggingHintsOnly) +{ + /// IcebergMetadata::totalRows must not return summary math. These fields stay populated for + /// mismatch warnings / equality fail-closed, but there is no getTotalRows shortcut anymore. + Iceberg::IcebergDataSnapshot snapshot; + snapshot.total_rows = 100; + snapshot.total_position_delete_rows = 10; + snapshot.total_equality_delete_rows = 0; + + EXPECT_EQ(*snapshot.total_rows, 100u); + EXPECT_EQ(*snapshot.total_position_delete_rows, 10u); + EXPECT_EQ(*snapshot.total_equality_delete_rows, 0u); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp new file mode 100644 index 000000000000..b863ee605446 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp @@ -0,0 +1,64 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include + +using namespace DB; + +TEST(IcebergDataObjectInfoClone, PreservesEqualityAndPositionDeletes) +{ + Iceberg::IcebergObjectSerializableInfo info; + info.data_object_file_path_key = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"); + info.file_format = "PARQUET"; + info.equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "s3://bucket/deletes/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1, 2}, + .schema_id = 7, + }); + info.position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "s3://bucket/deletes/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 42, + }); + + auto original = std::make_shared( + RelativePathWithMetadata{"data/file.parquet"}, info); + original->data_lake_metadata.emplace(); + original->data_lake_metadata->excluded_rows = std::make_shared(); + original->data_lake_metadata->excluded_rows->add(11); + + /// Object-slicing (the old ObjectIteratorSplitByBuckets path) drops Iceberg metadata. + ObjectInfo sliced_value = *original; + auto sliced = std::make_shared(sliced_value); + EXPECT_FALSE(std::dynamic_pointer_cast(sliced)); + ASSERT_TRUE(sliced->data_lake_metadata.has_value()); + ASSERT_TRUE(sliced->data_lake_metadata->excluded_rows); + EXPECT_EQ(sliced->data_lake_metadata->excluded_rows->size(), 1u); + + auto cloned = original->clone(); + auto iceberg_cloned = std::dynamic_pointer_cast(cloned); + ASSERT_TRUE(iceberg_cloned); + ASSERT_EQ(iceberg_cloned->info.equality_deletes_objects.size(), 1u); + EXPECT_EQ(iceberg_cloned->info.equality_deletes_objects[0].file_path, "s3://bucket/deletes/eq.parquet"); + ASSERT_TRUE(iceberg_cloned->info.equality_deletes_objects[0].equality_ids.has_value()); + EXPECT_EQ(iceberg_cloned->info.equality_deletes_objects[0].equality_ids->size(), 2u); + ASSERT_EQ(iceberg_cloned->info.position_deletes_objects.size(), 1u); + EXPECT_EQ(iceberg_cloned->info.position_deletes_objects[0].file_path, "s3://bucket/deletes/pos.parquet"); + EXPECT_EQ(iceberg_cloned->info.position_deletes_objects[0].sequence_number, 42); + ASSERT_TRUE(iceberg_cloned->data_lake_metadata.has_value()); + ASSERT_TRUE(iceberg_cloned->data_lake_metadata->excluded_rows); + EXPECT_EQ(iceberg_cloned->data_lake_metadata->excluded_rows->size(), 1u); + EXPECT_TRUE(iceberg_cloned->data_lake_metadata->excluded_rows->rb_contains(11)); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp new file mode 100644 index 000000000000..fa3442c4fad7 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp @@ -0,0 +1,90 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include + +#include + +using namespace DB; +using namespace DB::Iceberg; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ICEBERG_SPECIFICATION_VIOLATION; +} +} + +TEST(IcebergDeletionVectorPositions, AcceptsBoundaryPosition) +{ + /// Valid file-local positions are in [0, record_count). + const std::vector positions = {0, 9}; + EXPECT_NO_THROW(validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/2, /*data_file_record_count=*/10)); +} + +TEST(IcebergDeletionVectorPositions, RejectsPositionEqualToRecordCount) +{ + const std::vector positions = {10}; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/1, /*data_file_record_count=*/10); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("out of range"), std::string::npos); + } +} + +TEST(IcebergDeletionVectorPositions, RejectsPositionAboveRecordCount) +{ + const std::vector positions = {11}; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/1, /*data_file_record_count=*/10); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("out of range"), std::string::npos); + } +} + +TEST(IcebergDeletionVectorPositions, RejectsCardinalityExceedingRecordCount) +{ + const std::vector positions; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/11, /*data_file_record_count=*/10); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("exceeds data file record_count"), std::string::npos); + } +} + +TEST(IcebergDeletionVectorPositions, RejectsNegativeDataFileRecordCount) +{ + const std::vector positions; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/0, /*data_file_record_count=*/-1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("non-negative"), std::string::npos); + } +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp new file mode 100644 index 000000000000..ce5262728d1b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int NOT_IMPLEMENTED; +} +} + +TEST(IcebergParquetRowDeletesGuard, AcceptsParquet) +{ + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("parquet", "Deletion vectors")); + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("PARQUET", "Position deletes")); + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("Parquet", "Deletion vectors")); +} + +TEST(IcebergParquetRowDeletesGuard, RejectsNonParquet) +{ + try + { + Iceberg::requireParquetDataFileForRowDeletes("ORC", "Deletion vectors"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_IMPLEMENTED); + EXPECT_NE(e.message().find("Deletion vectors are only supported"), std::string::npos); + EXPECT_NE(e.message().find("ORC"), std::string::npos); + } + + try + { + Iceberg::requireParquetDataFileForRowDeletes("AVRO", "Position deletes"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_IMPLEMENTED); + EXPECT_NE(e.message().find("Position deletes are only supported"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp new file mode 100644 index 000000000000..dde10215881b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp @@ -0,0 +1,108 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include + +using namespace DB; +using namespace DB::Iceberg; + +namespace +{ + +ProcessedManifestFileEntryPtr makePositionDeleteEntry( + const String & file_format, + std::optional content_offset = std::nullopt, + std::optional content_size_in_bytes = std::nullopt) +{ + auto parsed = std::make_shared( + FileContentType::POSITION_DELETE, + IcebergPathFromMetadata::deserialize("s3://bucket/deletes/file"), + /*row_number=*/0, + ManifestEntryStatus::ADDED, + /*written_sequence_number=*/std::nullopt, + /*written_file_sequence_number=*/std::nullopt, + /*written_snapshot_id=*/std::nullopt, + DB::Row{}, + std::unordered_map{}, + std::unordered_map>{}, + file_format, + /*lower_reference_data_file_path=*/std::nullopt, + /*upper_reference_data_file_path=*/std::nullopt, + /*equality_ids=*/std::nullopt, + /*sort_order_id=*/std::nullopt, + /*record_count=*/1, + /*file_size_in_bytes=*/100, + content_offset, + content_size_in_bytes); + + auto processed = std::make_shared(); + processed->parsed_entry = std::move(parsed); + processed->common_partition_specification = std::make_shared(); + processed->sequence_number = 0; + processed->resolved_schema_id = 0; + processed->manifest_file_path = "s3://bucket/metadata/manifest.avro"; + return processed; +} + +} + +TEST(IcebergPositionDeleteKindPresence, Empty) +{ + const auto presence = getPositionDeleteKindPresence({}); + EXPECT_FALSE(presence.has_deletion_vectors); + EXPECT_FALSE(presence.has_parquet_position_deletes); + EXPECT_FALSE(presence.hasBoth()); +} + +TEST(IcebergPositionDeleteKindPresence, DeletionVectorsOnly) +{ + const auto presence = getPositionDeleteKindPresence({ + makePositionDeleteEntry("puffin", /*content_offset=*/4, /*content_size_in_bytes=*/40), + makePositionDeleteEntry("PUFFIN", /*content_offset=*/50, /*content_size_in_bytes=*/40), + }); + EXPECT_TRUE(presence.has_deletion_vectors); + EXPECT_FALSE(presence.has_parquet_position_deletes); + EXPECT_FALSE(presence.hasBoth()); +} + +TEST(IcebergPositionDeleteKindPresence, ParquetPositionDeletesOnly) +{ + const auto presence = getPositionDeleteKindPresence({ + makePositionDeleteEntry("parquet"), + makePositionDeleteEntry("puffin"), /// puffin without content offset/size is not a DV + }); + EXPECT_FALSE(presence.has_deletion_vectors); + EXPECT_TRUE(presence.has_parquet_position_deletes); + EXPECT_FALSE(presence.hasBoth()); +} + +TEST(IcebergPositionDeleteKindPresence, CoexistenceFailsClosedGate) +{ + const auto presence = getPositionDeleteKindPresence({ + makePositionDeleteEntry("puffin", /*content_offset=*/4, /*content_size_in_bytes=*/40), + makePositionDeleteEntry("parquet"), + }); + EXPECT_TRUE(presence.has_deletion_vectors); + EXPECT_TRUE(presence.has_parquet_position_deletes); + EXPECT_TRUE(presence.hasBoth()); +} + +TEST(IcebergPositionDeleteKindPresence, CrossManifestAggregation) +{ + /// Mimic totalRows: OR presence across manifests before deciding fail-closed. + auto first = getPositionDeleteKindPresence({ + makePositionDeleteEntry("puffin", /*content_offset=*/4, /*content_size_in_bytes=*/40), + }); + auto second = getPositionDeleteKindPresence({ + makePositionDeleteEntry("parquet"), + }); + + const bool has_deletion_vectors = first.has_deletion_vectors || second.has_deletion_vectors; + const bool has_parquet_position_deletes = first.has_parquet_position_deletes || second.has_parquet_position_deletes; + EXPECT_TRUE(has_deletion_vectors && has_parquet_position_deletes); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp new file mode 100644 index 000000000000..834ee6d42f6e --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp @@ -0,0 +1,306 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +extern const Event PuffinFileReadMicroseconds; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +struct ScopedPuffinFileReadProfileEvent +{ + ProfileEventTimeIncrement watch; + + ScopedPuffinFileReadProfileEvent() + : watch(ProfileEvents::PuffinFileReadMicroseconds) + { + ProfileEvents::increment(ProfileEvents::PuffinFilesRead); + } +}; + +constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; +constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; +constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; + +UInt32 readBigEndianUInt32(const UInt8 * data) +{ + return (static_cast(data[0]) << 24) + | (static_cast(data[1]) << 16) + | (static_cast(data[2]) << 8) + | static_cast(data[3]); +} + +UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) +{ + return (static_cast(key) << 32) | static_cast(sub_position); +} + +roaring::Roaring readRoaringPortableSafe(const char * data, size_t size, Int32 key) +{ + roaring::Roaring bitmap; + try + { + bitmap = roaring::Roaring::readSafe(data, size); + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to deserialize deletion vector roaring bitmap at key {}: {}", key, e.what()); + } + + /// `readSafe` only bounds the read; CRoaring requires internal validation before use on untrusted input. + const char * reason = nullptr; + if (!roaring::api::roaring_bitmap_internal_validate(&bitmap.roaring, &reason)) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector roaring bitmap at key {} failed internal validation: {}", + key, + reason ? reason : "unknown"); + } + + return bitmap; +} + +std::vector deserializeRoaringPositionBitmap(std::string_view bytes, std::optional expected_cardinality) +{ + if (bytes.size() < sizeof(Int64)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is too small"); + + const char * ptr = bytes.data(); + size_t remaining = bytes.size(); + + /// Iceberg deletion-vector roaring layout stores count and keys as little-endian. + const Int64 bitmap_count = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int64); + remaining -= sizeof(Int64); + + if (bitmap_count < 0 || bitmap_count > std::numeric_limits::max()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap count: {}", bitmap_count); + + std::vector positions; + Int32 last_key = -1; + Int32 remaining_count = static_cast(bitmap_count); + UInt64 running_cardinality = 0; + + while (remaining_count > 0) + { + if (remaining < sizeof(Int32)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is truncated while reading key"); + + const Int32 key = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int32); + remaining -= sizeof(Int32); + + if (key < 0 || key > DELETION_VECTOR_MAX_KEY) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap key: {}", key); + if (key <= last_key) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap keys must be sorted in ascending order"); + + auto bitmap = readRoaringPortableSafe(ptr, remaining, key); + + const size_t bitmap_size = bitmap.getSizeInBytes(/*portable=*/true); + if (bitmap_size > remaining) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector roaring bitmap at key {} exceeds blob size", key); + + if (expected_cardinality.has_value()) + { + const UInt64 bitmap_cardinality = bitmap.cardinality(); + UInt64 new_running_cardinality = 0; + if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality exceeds declared cardinality {}", + *expected_cardinality); + + if (new_running_cardinality > *expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds declared cardinality {}", + new_running_cardinality, + *expected_cardinality); + + running_cardinality = new_running_cardinality; + } + + for (UInt32 sub_position : bitmap) + { + const UInt64 position = positionFromKeyAndSubPosition(static_cast(key), sub_position); + if (position > static_cast(DELETION_VECTOR_MAX_POSITION)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector position {} is out of supported range", position); + positions.push_back(position); + } + + ptr += bitmap_size; + remaining -= bitmap_size; + last_key = key; + --remaining_count; + } + + if (remaining != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap has {} trailing bytes", remaining); + + if (expected_cardinality.has_value() && running_cardinality != *expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} does not match deserialized row count {}", + *expected_cardinality, + running_cardinality); + + return positions; +} + +std::string_view extractDeletionVectorPayload(std::string_view blob) +{ + if (blob.size() < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + const auto * blob_bytes = reinterpret_cast(blob.data()); + const UInt32 combined_length = readBigEndianUInt32(blob_bytes); + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + const size_t vector_size = combined_length - sizeof(DELETION_VECTOR_MAGIC); + const size_t expected_blob_size = sizeof(UInt32) + combined_length + sizeof(UInt32); + if (blob.size() != expected_blob_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob size {} does not match combined length {}", blob.size(), combined_length); + + if (std::memcmp(blob_bytes + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + const UInt8 * crc_input = blob_bytes + sizeof(UInt32); + const size_t crc_input_size = combined_length; + const UInt32 expected_crc = readBigEndianUInt32(blob_bytes + sizeof(UInt32) + combined_length); + const UInt32 actual_crc = static_cast(crc32_z(0L, reinterpret_cast(crc_input), crc_input_size)); + if (expected_crc != actual_crc) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector CRC mismatch"); + + return std::string_view(blob.data() + 2 * sizeof(UInt32), vector_size); +} + +} + +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context) +{ + if (offset < 0 || length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + if (offset > static_cast(file_size) || length > static_cast(file_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + Int64 end = 0; + if (common::addOverflow(offset, length, end)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + if (static_cast(end) > file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); +} + +void validateDeletionVectorEnvelope(const UInt8 * header, Int64 length) +{ + const UInt32 combined_length = readBigEndianUInt32(header); + if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + UInt64 expected_blob_size = 0; + if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + if (static_cast(length) != expected_blob_size) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob size {} does not match combined length {}", + length, + combined_length); +} + +std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality) +{ + if (expected_cardinality.has_value() && *expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + *expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + const std::string_view vector_bytes = extractDeletionVectorPayload(blob_bytes); + return deserializeRoaringPositionBitmap(vector_bytes, expected_cardinality); +} + +std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality) +{ + ScopedPuffinFileReadProfileEvent profile_event; + + if (length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob length is negative"); + + if (static_cast(length) > PUFFIN_DV_MAX_BLOB_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob length {} exceeds absolute limit {}", + length, + PUFFIN_DV_MAX_BLOB_SIZE); + + /// Reject before envelope peek / full allocate: cardinality alone is enough to fail closed. + if (expected_cardinality.has_value() && *expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + *expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + if (auto file_size = tryGetFileSizeFromReadBuffer(file)) + validatePuffinBlobBounds(offset, length, *file_size); + else if (offset < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin deletion vector offset {} or length {}", offset, length); + + if (static_cast(length) < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + auto * seekable = dynamic_cast(&file); + if (!seekable) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin deletion vector read requires a seekable buffer"); + + /// Peek combined_length + magic before allocating `length` (up to 2 GiB). Matches + /// `readDeletionVectorBlobBytes` in the SQL Puffin format path. + seekable->seek(offset, SEEK_SET); + + UInt8 header[8]; + file.readStrict(reinterpret_cast(header), sizeof(header)); + validateDeletionVectorEnvelope(header, length); + + String blob_data(static_cast(length), '\0'); + std::memcpy(blob_data.data(), header, sizeof(header)); + file.readStrict(blob_data.data() + sizeof(header), blob_data.size() - sizeof(header)); + + return deserializeDeletionVectorV1Blob(blob_data, expected_cardinality); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h new file mode 100644 index 000000000000..e025a7d39675 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace DB +{ + +/// Absolute cap on on-disk deletion-vector-v1 blob length before reading bytes into memory. +/// Aligns with Iceberg DeleteLoader's 2 GiB content-size check. +constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; +/// Absolute cap on materialized deleted positions (~800 MiB of UInt64s at this limit). +constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; + +/// Validate that [offset, offset + length) fits within file_size. +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context = "Puffin deletion vector"); + +/// Validate deletion-vector-v1 envelope (combined_length + magic) against declared blob `length`. +/// `header` must point at the first 8 bytes of the blob. Throws on mismatch before a full allocate. +void validateDeletionVectorEnvelope(const UInt8 * header, Int64 length); + +/// Deserialize a deletion-vector-v1 blob (magic + CRC wrapper + roaring bitmap payload). +std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality = std::nullopt); + +/// Read a deletion-vector-v1 blob from a Puffin file at the given offset and length. +std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality = std::nullopt); + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp new file mode 100644 index 000000000000..728937db4ae2 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp @@ -0,0 +1,643 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +extern const Event PuffinFileReadMicroseconds; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LZ4_DECODER_FAILED; +} + +namespace +{ + +struct ScopedPuffinFileReadProfileEvent +{ + ProfileEventTimeIncrement watch; + + ScopedPuffinFileReadProfileEvent() + : watch(ProfileEvents::PuffinFileReadMicroseconds) + { + ProfileEvents::increment(ProfileEvents::PuffinFilesRead); + } +}; + +constexpr UInt8 PUFFIN_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; +static_assert(sizeof(PUFFIN_MAGIC) == PUFFIN_MAGIC_SIZE); +constexpr UInt8 PUFFIN_FOOTER_COMPRESSED_FLAG = 0x01; +constexpr size_t PUFFIN_FOOTER_LZ4_MAX_RATIO = 255; +/// DV blob / materialization ceilings live in `PuffinDeletionVectorReader.h` (`PUFFIN_DV_MAX_*`) +/// and are shared with the Iceberg deletion-vector reader. Applied only when `deleted_rows` is +/// requested; subset reads that skip materialization do not enforce the materialization ceiling. +/// Intentionally not FormatSettings knobs: fail-closed amplification guards. + +void checkMagic(const UInt8 * p, const char * context) +{ + if (std::memcmp(p, PUFFIN_MAGIC, 4) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin magic ({})", context); +} + +/// `blob_region_end` is the end of the `Blob₁ ... Blobₙ` region, i.e. the offset of the footer's +/// leading magic. Blob offset/length must stay within [PUFFIN_MAGIC, blob_region_end), so a malformed +/// footer cannot point a blob into the footer payload or trailer. +void validatePuffinFooterBlobBounds(Int64 offset, Int64 length, size_t blob_region_end, size_t blob_index) +{ + if (offset < static_cast(sizeof(PUFFIN_MAGIC)) || length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); + + Int64 end = 0; + if (common::addOverflow(offset, length, end)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); + + if (static_cast(end) > blob_region_end) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); +} + +void validatePuffinFooterFlags(const UInt8 flags[4]) +{ + if (flags[1] != 0 || flags[2] != 0 || flags[3] != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); + if ((flags[0] & ~PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); +} + +String decompressPuffinFooterPayload(const char * data, size_t size) +{ + LZ4F_dctx * dctx = nullptr; + size_t ret = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to create LZ4 decompression context: {}", LZ4F_getErrorName(ret)); + + struct DecompressionContextGuard + { + LZ4F_dctx * dctx; + explicit DecompressionContextGuard(LZ4F_dctx * dctx_) : dctx(dctx_) { } + ~DecompressionContextGuard() { LZ4F_freeDecompressionContext(dctx); } + }; + DecompressionContextGuard guard(dctx); + + LZ4F_frameInfo_t frame_info{}; + size_t header_size = size; + ret = LZ4F_getFrameInfo(dctx, &frame_info, data, &header_size); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to read LZ4 frame info for Puffin footer: {}", LZ4F_getErrorName(ret)); + + const char * src = data + header_size; + size_t src_remaining = size - header_size; + + size_t max_by_ratio = 0; + if (common::mulOverflow(size, PUFFIN_FOOTER_LZ4_MAX_RATIO, max_by_ratio)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer compressed payload is too large"); + + if (frame_info.contentSize == 0) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame must declare content size"); + + const size_t max_decompressed_size = std::min(max_by_ratio, PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + if (frame_info.contentSize > max_decompressed_size) + { + if (max_decompressed_size == PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 content size {} exceeds absolute decompression limit {}", + frame_info.contentSize, + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 content size {} exceeds decompression limit {}", + frame_info.contentSize, + max_decompressed_size); + } + + String result; + result.resize(static_cast(frame_info.contentSize)); + + size_t dst_offset = 0; + while (true) + { + size_t src_read = src_remaining; + size_t dst_write = result.size() - dst_offset; + ret = LZ4F_decompress(dctx, result.data() + dst_offset, &dst_write, src, &src_read, nullptr); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to decompress Puffin footer: {}", LZ4F_getErrorName(ret)); + + src += src_read; + src_remaining -= src_read; + dst_offset += dst_write; + + if (ret == 0) + break; + + /// No forward progress while the frame is still open means the input is truncated: + /// LZ4F_decompress wants more input but we have given it everything. Stop to avoid spinning. + if (dst_write == 0 && src_read == 0) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame is incomplete"); + + if (dst_offset == result.size()) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer decompressed size exceeds declared content size {}", + frame_info.contentSize); + } + + if (dst_offset != frame_info.contentSize) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 decompressed size {} does not match content size {}", + dst_offset, + frame_info.contentSize); + + if (src_remaining != 0) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 frame has {} trailing bytes", + src_remaining); + + result.resize(dst_offset); + return result; +} + +void requireBlobMetadataField(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + if (!blob_obj->has(field_name) || blob_obj->isNull(field_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field '{}'", blob_index, field_name); +} + +/// Poco reports JSON booleans as integers (`std::numeric_limits::is_integer`). +bool isJSONInteger(const Poco::Dynamic::Var & value) +{ + return value.isInteger() && !value.isBoolean(); +} + +/// Poco stores JSON integers that do not fit signed Int64 as unsigned; convert() would throw RangeException. +std::optional tryJSONIntegerAsInt64(const Poco::Dynamic::Var & value) +{ + if (value.isInteger() && !value.isSigned()) + { + const auto as_uint64 = value.convert(); + if (as_uint64 > static_cast(std::numeric_limits::max())) + return std::nullopt; + return static_cast(as_uint64); + } + + return value.convert(); +} + +Int64 requireBlobMetadataInt64(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + requireBlobMetadataField(blob_obj, field_name, blob_index); + const Poco::Dynamic::Var & value = blob_obj->get(field_name); + if (!isJSONInteger(value)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be an integer", blob_index, field_name); + + auto as_int64 = tryJSONIntegerAsInt64(value); + if (!as_int64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' is out of Int64 range", blob_index, field_name); + return *as_int64; +} + +Int32 requireBlobMetadataFieldsElementInt32(const Poco::Dynamic::Var & value, size_t blob_index, size_t field_index) +{ + if (!isJSONInteger(value)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] must be an integer", blob_index, field_index); + + auto as_int64 = tryJSONIntegerAsInt64(value); + if (!as_int64) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int64 range", blob_index, field_index); + + if (*as_int64 < std::numeric_limits::min() || *as_int64 > std::numeric_limits::max()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int32 range", blob_index, field_index); + + return static_cast(*as_int64); +} + +String requireJSONStringValue(const Poco::Dynamic::Var & value, size_t blob_index, const char * field_name) +{ + if (!value.isString()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); + return value.extract(); +} + +String requireBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + requireBlobMetadataField(blob_obj, field_name, blob_index); + return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); +} + +String optBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + if (!blob_obj->has(field_name)) + return {}; + if (blob_obj->isNull(field_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); + return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); +} + +void requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) +{ + static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; + for (const char * key : required_properties) + { + auto it = blob.properties.find(key); + if (it == blob.properties.end() || it->second.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property '{}'", + blob_index, + key); + } + + UInt64 cardinality = 0; + if (!tryParse(cardinality, blob.properties.at("cardinality"))) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + blob_index); +} + +void parseStringValuedProperties( + const Poco::JSON::Object::Ptr & props_obj, + std::map * out, + bool for_blob, + size_t blob_index) +{ + for (const auto & [key, val] : *props_obj) + { + if (!val.isString()) + { + if (for_blob) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: property '{}' must be a string", blob_index, key); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer property '{}' must be a string", key); + } + if (out) + out->emplace(key, val.extract()); + } +} + +void parseBlobProperties(const Poco::JSON::Object::Ptr & blob_obj, PuffinBlob & blob, size_t blob_index, bool required) +{ + if (!blob_obj->has("properties")) + { + if (required) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field 'properties'", blob_index); + return; + } + + if (blob_obj->isNull("properties")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); + + auto props_obj = blob_obj->getObject("properties"); + if (!props_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); + + parseStringValuedProperties(props_obj, &blob.properties, /*for_blob=*/true, blob_index); +} + +/// Optional FileMetadata.properties: JSON object with string values (Iceberg Puffin spec). +/// Absent is allowed; present null / non-object is rejected. +void validateFileMetadataProperties(const Poco::JSON::Object::Ptr & footer_obj) +{ + if (!footer_obj->has("properties")) + return; + + if (footer_obj->isNull("properties")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); + + auto props_obj = footer_obj->getObject("properties"); + if (!props_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); + + parseStringValuedProperties(props_obj, /*out=*/nullptr, /*for_blob=*/false, /*blob_index=*/0); +} + +std::vector parseFooterJSON(const String & footer_json, size_t blob_region_end) +{ + Poco::JSON::Parser parser; + Poco::Dynamic::Var root; + try + { + root = parser.parse(footer_json); + } + catch (const Exception &) + { + /// catch `MEMORY_LIMIT_EXCEEDED` and other possible exceptions from `MemoryTracker` + throw; + } + catch (const Poco::Exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Puffin footer JSON: {}", e.displayText()); + } + + if (root.type() != typeid(Poco::JSON::Object::Ptr)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer JSON must be an object"); + + auto obj = root.extract(); + + validateFileMetadataProperties(obj); + + if (!obj->has("blobs") || obj->isNull("blobs")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer is missing required field 'blobs'"); + + auto blobs_arr = obj->getArray("blobs"); + if (!blobs_arr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'blobs' must be an array"); + + std::vector blobs; + for (size_t i = 0; i < blobs_arr->size(); ++i) + { + auto blob_obj = blobs_arr->getObject(static_cast(i)); + if (!blob_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: must be an object", i); + + PuffinBlob blob; + + blob.type = requireBlobMetadataString(blob_obj, "type", i); + blob.snapshot_id = requireBlobMetadataInt64(blob_obj, "snapshot-id", i); + blob.sequence_number = requireBlobMetadataInt64(blob_obj, "sequence-number", i); + blob.offset = requireBlobMetadataInt64(blob_obj, "offset", i); + blob.length = requireBlobMetadataInt64(blob_obj, "length", i); + + if (blob.type == "deletion-vector-v1") + { + if (blob.snapshot_id != -1 || blob.sequence_number != -1) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 snapshot-id and sequence-number must be -1", + i); + + /// Spec requires the key to be omitted; present null is not omission. + if (blob_obj->has("compression-codec")) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 must omit 'compression-codec'", + i); + + parseBlobProperties(blob_obj, blob, i, /*required=*/true); + requireDeletionVectorV1Properties(blob, i); + } + else + { + blob.compression_codec = optBlobMetadataString(blob_obj, "compression-codec", i); + parseBlobProperties(blob_obj, blob, i, /*required=*/false); + } + + requireBlobMetadataField(blob_obj, "fields", i); + auto fields_arr = blob_obj->getArray("fields"); + if (!fields_arr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'fields' must be an array", i); + + for (size_t j = 0; j < fields_arr->size(); ++j) + blob.fields.push_back(requireBlobMetadataFieldsElementInt32(fields_arr->get(static_cast(j)), i, j)); + + validatePuffinFooterBlobBounds(blob.offset, blob.length, blob_region_end, i); + + blobs.push_back(std::move(blob)); + } + return blobs; +} + +} // namespace + +void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size) +{ + if (out.size() > max_buffered_size) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable buffer size {} exceeds absolute limit {}", + out.size(), + max_buffered_size); + } + + std::vector tmp(DBMS_DEFAULT_BUFFER_SIZE); + while (!buf.eof()) + { + const size_t capacity = max_buffered_size - out.size(); + if (capacity == 0) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", + max_buffered_size); + } + + const size_t to_read = std::min(tmp.size(), capacity); + const size_t n = buf.read(reinterpret_cast(tmp.data()), to_read); + if (n == 0) + break; + + out.insert(out.end(), tmp.data(), tmp.data() + n); + + /// If we filled the remaining capacity and the stream still has data, fail closed. + if (n == capacity && !buf.eof()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", + max_buffered_size); + } + } +} + +std::vector readPuffinFooterBlobsFromSeekable(SeekableReadBuffer & seekable, size_t file_size) +{ + ScopedPuffinFileReadProfileEvent profile_event; + + if (file_size < 16) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); + + seekable.seek(0, SEEK_SET); + char magic_buf[4]; + seekable.readStrict(magic_buf, 4); + checkMagic(reinterpret_cast(magic_buf), "header"); + + seekable.seek(static_cast(file_size - PUFFIN_FOOTER_TRAILER_SIZE), SEEK_SET); + Int32 footer_length_signed = 0; + readBinaryLittleEndian(footer_length_signed, seekable); + + UInt8 flags[4] = {}; + seekable.readStrict(reinterpret_cast(flags), sizeof(flags)); + + char trailing_buf[4]; + seekable.readStrict(trailing_buf, 4); + checkMagic(reinterpret_cast(trailing_buf), "trailing"); + validatePuffinFooterFlags(flags); + + if (footer_length_signed <= 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + UInt64 footer_trailer_size = 0; + if (common::addOverflow( + static_cast(footer_length_signed), + static_cast(PUFFIN_FOOTER_TRAILER_SIZE), + footer_trailer_size) + || footer_trailer_size > file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + const size_t footer_length = static_cast(footer_length_signed); + const size_t payload_start = file_size - PUFFIN_FOOTER_TRAILER_SIZE - footer_length; + /// Footer layout is Magic | FooterPayload | …; require a distinct footer-open Magic after the + /// file header Magic (payload_start == 4 would reuse the header Magic when blobs is empty). + if (payload_start < 2 * sizeof(PUFFIN_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + if (footer_length > PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin footer payload size {} exceeds absolute limit {}", + footer_length, + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + + seekable.seek(static_cast(payload_start - sizeof(PUFFIN_MAGIC)), SEEK_SET); + char leading_magic_buf[4]; + seekable.readStrict(leading_magic_buf, 4); + checkMagic(reinterpret_cast(leading_magic_buf), "footer"); + + String footer_payload(footer_length, '\0'); + seekable.seek(static_cast(payload_start), SEEK_SET); + seekable.readStrict(footer_payload.data(), footer_length); + + String footer_json; + if ((flags[0] & PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) + footer_json = decompressPuffinFooterPayload(footer_payload.data(), footer_payload.size()); + else + footer_json = std::move(footer_payload); + + const size_t blob_region_end = payload_start - sizeof(PUFFIN_MAGIC); + return parseFooterJSON(footer_json, blob_region_end); +} + +const PuffinBlob & bindDeletionVectorBlob( + const std::vector & blobs, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view expected_referenced_data_file, + UInt64 expected_cardinality) +{ + const PuffinBlob * matched = nullptr; + size_t matched_index = 0; + + for (size_t i = 0; i < blobs.size(); ++i) + { + if (blobs[i].offset != content_offset || blobs[i].length != content_size_in_bytes) + continue; + + if (matched) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Multiple Puffin blobs claim offset {} length {}", + content_offset, + content_size_in_bytes); + } + + matched = &blobs[i]; + matched_index = i; + } + + if (!matched) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "No Puffin footer blob at offset {} length {}", + content_offset, + content_size_in_bytes); + } + + if (matched->type != "deletion-vector-v1") + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} at offset {} length {} has type '{}', expected deletion-vector-v1", + matched_index, + content_offset, + content_size_in_bytes, + matched->type); + } + + if (!matched->compression_codec.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 must omit compression-codec", + matched_index); + } + + const auto ref_it = matched->properties.find("referenced-data-file"); + if (ref_it == matched->properties.end() || ref_it->second.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property 'referenced-data-file'", + matched_index); + } + + if (ref_it->second != expected_referenced_data_file) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} referenced-data-file '{}' does not match expected data file '{}'", + matched_index, + ref_it->second, + expected_referenced_data_file); + } + + const auto card_it = matched->properties.find("cardinality"); + if (card_it == matched->properties.end() || card_it->second.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property 'cardinality'", + matched_index); + } + + UInt64 footer_cardinality = 0; + if (!tryParse(footer_cardinality, card_it->second)) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + matched_index); + } + + if (footer_cardinality != expected_cardinality) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} cardinality {} does not match expected cardinality {}", + matched_index, + footer_cardinality, + expected_cardinality); + } + + return *matched; +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFile.h b/src/Storages/ObjectStorage/DataLakes/PuffinFile.h new file mode 100644 index 000000000000..8d340271a4d1 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFile.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +/// Leading / footer-open magic is always 4 bytes (`PFA1`). +constexpr size_t PUFFIN_MAGIC_SIZE = 4; +constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; +/// Absolute cap on footer payload size (uncompressed JSON bytes, or declared LZ4 contentSize). +constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; + +/// Non-seekable SQL path must buffer the whole file to reach the trailer. Cap total buffered size +/// so a crafted pipe cannot allocate unbounded memory before footer-length validation. +/// Sized for one max DV blob + max footer (header magic + blob + footer magic + payload + trailer). +constexpr size_t PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE = PUFFIN_MAGIC_SIZE + PUFFIN_DV_MAX_BLOB_SIZE + + PUFFIN_MAGIC_SIZE + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE + PUFFIN_FOOTER_TRAILER_SIZE; + +struct PuffinBlob +{ + String type; + Int64 snapshot_id = 0; + Int64 sequence_number = 0; + std::vector fields; + Int64 offset = 0; + Int64 length = 0; + String compression_codec; + std::map properties; +}; + +/// Append bytes from `buf` into `out` until EOF. Throws if `out` would exceed `max_buffered_size`. +void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size); + +/// Parse the Puffin footer of a seekable file and return blob metadata. +std::vector readPuffinFooterBlobsFromSeekable(SeekableReadBuffer & seekable, size_t file_size); + +/// Find the unique footer blob at (`content_offset`, `content_size_in_bytes`) and bind it as a +/// deletion-vector-v1 for `expected_referenced_data_file` with `expected_cardinality`. +/// Throws if the slice is missing, ambiguous, or does not match the expected DV identity. +const PuffinBlob & bindDeletionVectorBlob( + const std::vector & blobs, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view expected_referenced_data_file, + UInt64 expected_cardinality); + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp new file mode 100644 index 000000000000..d3745cb2c12e --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp @@ -0,0 +1,128 @@ +#include + +#include +#include +#include + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheWeightLost; +} + +namespace DB +{ + +DataLakeObjectMetadata::ExcludedRowsPtr PuffinFilesCache::cloneExcludedRows(const PuffinFilesCacheCell & cell) +{ + if (cell.is_empty_deletion_vector) + return nullptr; + + auto cloned = std::make_shared(); + cloned->merge(*cell.excluded_rows); + return cloned; +} + +bool PuffinFilesCacheKey::operator==(const PuffinFilesCacheKey & other) const +{ + return storage_identity == other.storage_identity + && file_path == other.file_path + && etag == other.etag + && content_offset == other.content_offset + && content_size_in_bytes == other.content_size_in_bytes + && referenced_data_file == other.referenced_data_file + && expected_cardinality == other.expected_cardinality + && data_file_record_count == other.data_file_record_count; +} + +size_t PuffinFilesCacheKeyHash::operator()(const PuffinFilesCacheKey & key) const +{ + size_t hash = 0; + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.storage_identity.data(), key.storage_identity.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.file_path.data(), key.file_path.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.etag.data(), key.etag.size())); + boost::hash_combine(hash, key.content_offset); + boost::hash_combine(hash, key.content_size_in_bytes); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.referenced_data_file.data(), key.referenced_data_file.size())); + boost::hash_combine(hash, key.expected_cardinality); + boost::hash_combine(hash, key.data_file_record_count); + return hash; +} + +UInt64 PuffinFilesCacheCell::calculateMemorySize(bool is_empty_deletion_vector_, const DataLakeObjectMetadata::ExcludedRowsPtr & excluded_rows_) +{ + if (is_empty_deletion_vector_) + return EMPTY_DELETION_VECTOR_WEIGHT; + + return (excluded_rows_ ? excluded_rows_->getAllocatedBytes() : 0) + SIZE_IN_MEMORY_OVERHEAD; +} + +PuffinFilesCacheCell::PuffinFilesCacheCell(DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows_) + : excluded_rows(std::move(excluded_rows_)) + , is_empty_deletion_vector(!excluded_rows) + , memory_bytes(calculateMemorySize(is_empty_deletion_vector, excluded_rows)) +{ +} + +size_t PuffinFilesCacheWeightFunction::operator()(const PuffinFilesCacheCell & cell) const +{ + return cell.memory_bytes; +} + +PuffinFilesCache::PuffinFilesCache( + const String & cache_policy, + size_t max_size_in_bytes, + size_t max_count, + double size_ratio) + : Base( + cache_policy, + CurrentMetrics::PuffinFilesCacheBytes, + CurrentMetrics::PuffinFilesCacheFiles, + max_size_in_bytes, + max_count, + size_ratio) + , log(getLogger("PuffinFilesCache")) +{ +} + +String PuffinFilesCache::makeStorageIdentity(const IObjectStorage & object_storage) +{ + return object_storage.getName() + "://" + object_storage.getObjectsNamespace() + "/" + object_storage.getCommonKeyPrefix(); +} + +std::optional PuffinFilesCache::tryCreateKey( + const String & storage_identity, + const String & file_path, + const String & etag, + Int64 content_offset, + Int64 content_size_in_bytes, + const String & referenced_data_file, + UInt64 expected_cardinality, + UInt64 data_file_record_count) +{ + if (etag.empty()) + return std::nullopt; + + return PuffinFilesCacheKey{ + storage_identity, + file_path, + etag, + content_offset, + content_size_in_bytes, + referenced_data_file, + expected_cardinality, + data_file_record_count}; +} + +void PuffinFilesCache::onEntryRemoval(const size_t weight_loss, const MappedPtr &) +{ + LOG_TRACE(log, "Puffin files cache eviction"); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheWeightLost, weight_loss); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h new file mode 100644 index 000000000000..8e49594e928b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h @@ -0,0 +1,212 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheHits; +extern const Event PuffinFilesCacheMisses; +extern const Event PuffinFilesCacheWeightLost; +} + +namespace DB +{ + +class IObjectStorage; + +struct PuffinFilesCacheKey +{ + /// Distinguishes object-storage backends that share the same relative path (and possibly etag). + /// Built via `makeStorageIdentity` from storage type + namespace + common key prefix. + String storage_identity; + String file_path; + String etag; + Int64 content_offset = 0; + Int64 content_size_in_bytes = 0; + String referenced_data_file; + /// Manifest DV record_count / expected roaring cardinality. Included so a cache hit cannot + /// skip re-validation when a later request declares a different cardinality for the same slice. + UInt64 expected_cardinality = 0; + /// Data-file manifest record_count used to bound DV positions. Same rationale as cardinality. + UInt64 data_file_record_count = 0; + + bool operator==(const PuffinFilesCacheKey & other) const; +}; + +struct PuffinFilesCacheKeyHash +{ + size_t operator()(const PuffinFilesCacheKey & key) const; +}; + +struct PuffinFilesCacheCell : private boost::noncopyable +{ + DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows; + bool is_empty_deletion_vector = false; + UInt64 memory_bytes = 0; + + explicit PuffinFilesCacheCell(DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows_); + +private: + static constexpr UInt64 EMPTY_DELETION_VECTOR_WEIGHT = 1; + static constexpr size_t SIZE_IN_MEMORY_OVERHEAD = 200; + + static UInt64 calculateMemorySize(bool is_empty_deletion_vector_, const DataLakeObjectMetadata::ExcludedRowsPtr & excluded_rows_); +}; + +struct PuffinFilesCacheWeightFunction +{ + size_t operator()(const PuffinFilesCacheCell & cell) const; +}; + +/// Cache for parsed content loaded from Puffin files (deletion vectors today, indexes later). +class PuffinFilesCache : public CacheBase +{ +public: + using Base = CacheBase; + + PuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_count, double size_ratio); + + /// Stable backend identity for cache keys: `getName()://getObjectsNamespace()/getCommonKeyPrefix()`. + static String makeStorageIdentity(const IObjectStorage & object_storage); + + static std::optional tryCreateKey( + const String & storage_identity, + const String & file_path, + const String & etag, + Int64 content_offset, + Int64 content_size_in_bytes, + const String & referenced_data_file, + UInt64 expected_cardinality, + UInt64 data_file_record_count); + + template + DataLakeObjectMetadata::ExcludedRowsPtr getOrSetDeletionVector(const PuffinFilesCacheKey & key, LoadFunc && load_fn) + { + /// True if this caller's load_fn ran. Used only for tracing; hit/miss metrics use the + /// atomic CacheGetOrSetOutcome from getOrSetWithOutcome (not a follow-up contains()). + bool loaded = false; + auto load_fn_wrapper = [&]() + { + loaded = true; + auto excluded_rows = load_fn(); + const bool is_empty_deletion_vector = !excluded_rows; + if (is_empty_deletion_vector) + { + LOG_TRACE( + log, + "Cached empty puffin deletion vector for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else + { + LOG_TRACE( + log, + "Loaded puffin deletion vector into cache for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + return std::make_shared(std::move(excluded_rows)); + }; + + auto [cell, outcome] = Base::getOrSetWithOutcome(key, load_fn_wrapper); + const bool served_from_cache = outcome == CacheGetOrSetOutcome::Hit; + if (!served_from_cache) + { + if (loaded && outcome == CacheGetOrSetOutcome::MissNotResident) + { + LOG_TRACE( + log, + "Puffin files cache miss (load discarded by concurrent clear) for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else if (!loaded && outcome == CacheGetOrSetOutcome::MissNotResident) + { + LOG_TRACE( + log, + "Puffin files cache miss (waited for load discarded by concurrent clear) for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else + { + LOG_TRACE( + log, + "Puffin files cache miss for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheMisses); + } + else if (cell->is_empty_deletion_vector) + { + LOG_TRACE( + log, + "Puffin files cache hit (empty deletion vector) for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheHits); + } + else + { + LOG_TRACE( + log, + "Puffin files cache hit for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheHits); + } + + return cloneExcludedRows(*cell); + } + +private: + static DataLakeObjectMetadata::ExcludedRowsPtr cloneExcludedRows(const PuffinFilesCacheCell & cell); + + LoggerPtr log; + + void onEntryRemoval(size_t weight_loss, const MappedPtr &) override; +}; + +using PuffinFilesCachePtr = std::shared_ptr; + +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp new file mode 100644 index 000000000000..1d17d21d7d2b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp @@ -0,0 +1,68 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +Chunk makeChunkWithFileRowNumbers(const std::vector & values, size_t row_num_offset = 0) +{ + auto column = ColumnUInt64::create(); + for (UInt64 value : values) + column->insert(value); + + Chunk chunk(Columns{std::move(column)}, values.size()); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +/// Mimics Iceberg equality-delete `FilterTransform`: shrink columns, leave `applied_filter` unset. +void shrinkWithoutAppliedFilter(Chunk & chunk, const IColumn::Filter & filter) +{ + size_t result_size = 0; + for (UInt8 keep : filter) + result_size += keep != 0; + + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->filter(filter, -1); + chunk.setColumns(std::move(columns), result_size); +} + +std::vector readValues(const Chunk & chunk) +{ + const auto & column = assert_cast(*chunk.getColumns().at(0)); + std::vector values; + values.reserve(column.size()); + for (size_t i = 0; i < column.size(); ++i) + values.push_back(column.getData()[i]); + return values; +} + +std::shared_ptr makeExcludedRows(std::initializer_list positions) +{ + auto excluded_rows = std::make_shared(); + for (UInt64 position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +/// Equality removes file row 1; DV deletes file position 2. Survivors must be {0, 3}. +TEST(DeletionVectorBeforeEqualityFilter, CorrectOrderKeepsFileRowMapping) +{ + Chunk chunk = makeChunkWithFileRowNumbers({0, 1, 2, 3}); + + DeletionVectorTransform::transform(chunk, *makeExcludedRows({2})); + ASSERT_EQ(readValues(chunk), (std::vector{0, 1, 3})); + + /// Drop equality-deleted value 1 without updating applied_filter (as FilterTransform does). + shrinkWithoutAppliedFilter(chunk, IColumn::Filter{1, 0, 1}); + EXPECT_EQ(readValues(chunk), (std::vector{0, 3})); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp new file mode 100644 index 000000000000..1a5c091e819d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp @@ -0,0 +1,124 @@ +#include + +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +Chunk makeConstCountChunk(size_t num_rows, size_t row_num_offset) +{ + auto nested = ColumnUInt64::create(); + nested->insertDefault(); + Columns columns; + columns.emplace_back(ColumnConst::create(std::move(nested), num_rows)); + Chunk chunk(std::move(columns), num_rows); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +Chunk makeMaterializedChunk(const std::vector & values, size_t row_num_offset = 0) +{ + auto column = ColumnUInt64::create(); + for (UInt64 value : values) + column->insert(value); + + Chunk chunk(Columns{std::move(column)}, values.size()); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +std::shared_ptr makeExcludedRows(std::initializer_list positions) +{ + auto excluded_rows = std::make_shared(); + for (UInt64 position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(RoaringBitmapRangeCardinality, CountsInclusiveStartExclusiveEnd) +{ + DataLakeObjectMetadata::ExcludedRows bitmap; + bitmap.add(1); + bitmap.add(5); + bitmap.add(10); + + EXPECT_EQ(bitmap.rb_range_cardinality(0, 0), 0u); + EXPECT_EQ(bitmap.rb_range_cardinality(5, 5), 0u); + EXPECT_EQ(bitmap.rb_range_cardinality(0, 5), 1u); + EXPECT_EQ(bitmap.rb_range_cardinality(0, 6), 2u); + EXPECT_EQ(bitmap.rb_range_cardinality(1, 11), 3u); + EXPECT_EQ(bitmap.rb_range_cardinality(2, 10), 1u); + EXPECT_EQ(bitmap.rb_range_cardinality(11, 100), 0u); +} + +TEST(RoaringBitmapRangeCardinality, MatchesRbRangeCountForLargeBitmap) +{ + DataLakeObjectMetadata::ExcludedRows bitmap; + /// Force large roaring path (small-set threshold is 32). + for (UInt64 i = 0; i < 64; ++i) + bitmap.add(i * 3); + ASSERT_TRUE(bitmap.isLarge()); + + DataLakeObjectMetadata::ExcludedRows subset; + const UInt64 via_range = bitmap.rb_range(10, 100, subset); + EXPECT_EQ(bitmap.rb_range_cardinality(10, 100), via_range); + EXPECT_EQ(subset.size(), via_range); +} + +TEST(RoaringBitmapRangeCardinality, SumOfRowGroupRangesMatchesWholeFile) +{ + DataLakeObjectMetadata::ExcludedRows bitmap; + for (UInt64 i = 0; i < 10'000; ++i) + bitmap.add(i * 7); + ASSERT_TRUE(bitmap.isLarge()); + + constexpr size_t rows_per_group = 1'000; + constexpr size_t num_groups = 200; + UInt64 summed = 0; + for (size_t group = 0; group < num_groups; ++group) + { + const UInt64 start = group * rows_per_group; + summed += bitmap.rb_range_cardinality(start, start + rows_per_group); + } + + EXPECT_EQ(summed, bitmap.rb_range_cardinality(0, num_groups * rows_per_group)); + EXPECT_EQ(summed, bitmap.size()); +} + +TEST(DeletionVectorNeedOnlyCount, ConstChunkUsesRangeCardinality) +{ + /// Large enough that a dense Filter would be expensive; const columns stay O(1). + constexpr size_t num_rows = 5'000'000; + Chunk chunk = makeConstCountChunk(num_rows, /*row_num_offset=*/0); + + DeletionVectorTransform::transform(chunk, *makeExcludedRows({0, 1, num_rows - 1, num_rows + 10})); + EXPECT_EQ(chunk.getNumRows(), num_rows - 3); + ASSERT_EQ(chunk.getNumColumns(), 1u); + EXPECT_TRUE(isColumnConst(*chunk.getColumns()[0])); + EXPECT_FALSE(chunk.getChunkInfos().get()->applied_filter.has_value()); +} + +TEST(DeletionVectorNeedOnlyCount, ConstChunkHonorsRowNumOffset) +{ + Chunk chunk = makeConstCountChunk(/*num_rows=*/10, /*row_num_offset=*/100); + /// Deletes at absolute positions 99 (before), 105 (inside), 110 (end exclusive / outside). + DeletionVectorTransform::transform(chunk, *makeExcludedRows({99, 105, 110})); + EXPECT_EQ(chunk.getNumRows(), 9u); +} + +TEST(DeletionVectorNeedOnlyCount, MaterializedChunkStillUsesDenseFilter) +{ + Chunk chunk = makeMaterializedChunk({10, 11, 12, 13}, /*row_num_offset=*/10); + DeletionVectorTransform::transform(chunk, *makeExcludedRows({11, 13})); + EXPECT_EQ(chunk.getNumRows(), 2u); + ASSERT_TRUE(chunk.getChunkInfos().get()->applied_filter.has_value()); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp new file mode 100644 index 000000000000..9903493c3413 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp @@ -0,0 +1,28 @@ +#include + +#include +#include + +using namespace DB; + +TEST(HasNonEmptyExcludedRows, EmptyOptionalIsFalse) +{ + EXPECT_FALSE(hasNonEmptyExcludedRows(std::nullopt)); +} + +TEST(HasNonEmptyExcludedRows, MissingOrEmptyBitmapIsFalse) +{ + DataLakeObjectMetadata metadata; + EXPECT_FALSE(hasNonEmptyExcludedRows(metadata)); + + metadata.excluded_rows = std::make_shared(); + EXPECT_FALSE(hasNonEmptyExcludedRows(metadata)); +} + +TEST(HasNonEmptyExcludedRows, NonEmptyBitmapIsTrue) +{ + DataLakeObjectMetadata metadata; + metadata.excluded_rows = std::make_shared(); + metadata.excluded_rows->add(7); + EXPECT_TRUE(hasNonEmptyExcludedRows(metadata)); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp new file mode 100644 index 000000000000..0c036309b73b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +TEST(PuffinDeletionVectorBounds, RejectsLengthExceedingFileSize) +{ + const String data(64, '\0'); + ReadBufferFromOutsideMemoryFile file("test.puffin", data); + + try + { + readDeletionVectorFromPuffin(file, 0, 1000); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorBounds, RejectsOffsetPlusLengthOverflow) +{ + const String data(64, '\0'); + ReadBufferFromOutsideMemoryFile file("test.puffin", data); + + try + { + readDeletionVectorFromPuffin(file, 60, 12); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorBounds, ValidatePuffinBlobBoundsRejectsNegativeOffset) +{ + try + { + validatePuffinBlobBounds(-1, 10, 64); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp new file mode 100644 index 000000000000..fbc7bb77ddff --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp @@ -0,0 +1,95 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +constexpr Int64 large_declared_length = 64 * 1024 * 1024; + +} + +TEST(PuffinDeletionVectorCardinality, RejectsCardinalityAboveMaterializationLimitBeforeParse) +{ + /// Ceiling is checked before payload validation, so even an empty blob must fail closed. + try + { + deserializeDeletionVectorV1Blob(std::string_view{}, PUFFIN_DV_MAX_MATERIALIZED_POSITIONS + 1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds materialization limit"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, RejectsCardinalityAboveLimitBeforeFullAllocate) +{ + /// Huge declared length with only a tiny buffer: without an early ceiling check this would + /// allocate `large_declared_length` (or fail mid-read after that allocate). ReadBufferFromMemory + /// does not expose file size, so bounds checks alone do not stop this. + const char header[8] = {}; + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin( + file, 0, large_declared_length, PUFFIN_DV_MAX_MATERIALIZED_POSITIONS + 1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds materialization limit"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, RejectsBitmapExceedingDeclaredCardinality) +{ + const std::string_view blob( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)); + + try + { + deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds declared cardinality"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, AcceptsMatchingCardinality) +{ + const std::string_view blob( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)); + + const auto positions = deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp new file mode 100644 index 000000000000..4db35bad5ad4 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp @@ -0,0 +1,124 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +/// Large declared length that would force a huge allocate if peeked after full read. +constexpr Int64 large_declared_length = 64 * 1024 * 1024; + +} + +TEST(PuffinDeletionVectorEnvelope, RejectsLengthBelowEnvelopeMinimum) +{ + const char zeros[16] = {}; + ReadBufferFromMemory file(zeros, sizeof(zeros)); + + try + { + readDeletionVectorFromPuffin(file, 0, 11); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("too small"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, RejectsInvalidMagicBeforeFullAllocate) +{ + /// Only 8 bytes available; declared length is huge. Without envelope peek this would allocate + /// `large_declared_length` (or fail mid-read after that allocate). ReadBufferFromMemory does not + /// expose file size, so the absolute 2 GiB / bounds checks alone do not stop this. + const UInt8 header[8] = {0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00}; // wrong magic + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin(file, 0, large_declared_length); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("Invalid deletion vector magic"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, RejectsCombinedLengthMismatchBeforeFullAllocate) +{ + /// Valid magic, but combined_length implies blob size 0x24+8=44, while caller length is huge. + const UInt8 header[8] = {0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64}; + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin(file, 0, large_declared_length); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("does not match combined length"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, ReadsValidBlobAfterEnvelopePeek) +{ + ReadBufferFromMemory file(two_position_dv_blob, sizeof(two_position_dv_blob)); + const auto positions = readDeletionVectorFromPuffin( + file, 0, static_cast(sizeof(two_position_dv_blob)), /*expected_cardinality=*/2); + + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(PuffinDeletionVectorEnvelope, RejectsInternallyInconsistentRoaringAfterReadSafe) +{ + /// Valid DV envelope + CRC wrapping a portable roaring that `readSafe` accepts but + /// `roaring_bitmap_internal_validate` rejects (duplicate values in an array container). + /// Bytes from croaring's robust_deserialization_unit `deserialize_unsorted_array` fixture. + constexpr UInt8 inconsistent_roaring_dv_blob[] = { + 0x00, 0x00, 0x00, 0x1D, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3B, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x6E, 0x0E, 0x9B, 0x12, + }; + + const std::string_view blob( + reinterpret_cast(inconsistent_roaring_dv_blob), sizeof(inconsistent_roaring_dv_blob)); + + try + { + deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("failed internal validation"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp new file mode 100644 index 000000000000..43242b96bb36 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp @@ -0,0 +1,126 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// Two equal-cardinality deletion-vector-v1 blobs for different data files. +/// Blob A at offset 4 length 44 -> /data/file_a.parquet positions {2, 5} +/// Blob B at offset 48 length 44 -> /data/file_b.parquet positions {7, 9} +constexpr UInt8 two_equal_cardinality_dvs_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x09, 0x00, 0xB7, 0xB0, 0x20, 0xFF, 0x50, 0x46, 0x41, 0x31, + 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x7B, 0x22, 0x74, 0x79, 0x70, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, + 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, + 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, + 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, + 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, + 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, + 0x61, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, + 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, + 0x2C, 0x20, 0x7B, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, + 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, + 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, + 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, + 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, + 0x22, 0x3A, 0x20, 0x34, 0x38, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, + 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, + 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, + 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, 0x62, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, + 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, + 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, 0x5D, 0x7D, 0x9A, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +std::vector readFixtureBlobs() +{ + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + return readPuffinFooterBlobsFromSeekable(file, sizeof(two_equal_cardinality_dvs_puffin)); +} + +} + +TEST(PuffinDeletionVectorFooterBind, AcceptsMatchingOffsetReferencedFileAndCardinality) +{ + const auto blobs = readFixtureBlobs(); + const auto & blob = bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + EXPECT_EQ(blob.type, "deletion-vector-v1"); + EXPECT_EQ(blob.properties.at("referenced-data-file"), "/data/file_a.parquet"); + + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + const auto positions = readDeletionVectorFromPuffin(file, 4, 44, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(PuffinDeletionVectorFooterBind, RejectsSwappedOffsetForOtherFilesEqualCardinalityBlob) +{ + /// Manifest for file A points at file B's slice (same length and cardinality). + const auto blobs = readFixtureBlobs(); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/48, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("referenced-data-file"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorFooterBind, RejectsCardinalityMismatch) +{ + const auto blobs = readFixtureBlobs(); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 3); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("cardinality"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorFooterBind, RejectsMissingOffset) +{ + const auto blobs = readFixtureBlobs(); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/99, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("No Puffin footer blob"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp new file mode 100644 index 000000000000..dba4b03f5244 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#if USE_AVRO + +#include +#include +#include + +using namespace DB; +using namespace DB::Iceberg; + +namespace DB::ErrorCodes +{ +extern const int ICEBERG_SPECIFICATION_VIOLATION; +} + +TEST(PuffinDeletionVectorReferencedDataFile, AcceptsNonEmptyDirectField) +{ + const auto path = IcebergPathFromMetadata::deserialize("/data/file.parquet"); + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + EXPECT_NO_THROW(requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/true, path, manifest)); +} + +TEST(PuffinDeletionVectorReferencedDataFile, RejectsBoundsOnlyFallback) +{ + const auto path = IcebergPathFromMetadata::deserialize("/data/file.parquet"); + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + try + { + requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/false, path, manifest); + FAIL() << "Expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_TRUE(e.message().find("referenced_data_file") != std::string::npos); + } +} + +TEST(PuffinDeletionVectorReferencedDataFile, RejectsMissingPath) +{ + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + try + { + requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/true, std::nullopt, manifest); + FAIL() << "Expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + } +} + +TEST(PuffinDeletionVectorReferencedDataFile, RejectsEmptyPath) +{ + const auto empty_path = IcebergPathFromMetadata::deserialize(""); + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + try + { + requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/true, empty_path, manifest); + FAIL() << "Expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + } +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp new file mode 100644 index 000000000000..c48b365bc933 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp @@ -0,0 +1,89 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheClone, CacheHitReturnsIndependentCopy) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 3, 100); + ASSERT_TRUE(key.has_value()); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({1, 5, 10}); + }; + + auto first = cache.getOrSetDeletionVector(*key, load_fn); + auto second = cache.getOrSetDeletionVector(*key, load_fn); + + ASSERT_EQ(load_calls, 1); + ASSERT_NE(first, second); + EXPECT_TRUE(first->rb_contains(1)); + EXPECT_TRUE(first->rb_contains(5)); + EXPECT_TRUE(first->rb_contains(10)); + EXPECT_FALSE(first->rb_contains(99)); + + first->add(99); + + EXPECT_FALSE(second->rb_contains(99)); + + auto third = cache.getOrSetDeletionVector(*key, load_fn); + ASSERT_NE(third, first); + EXPECT_FALSE(third->rb_contains(99)); + EXPECT_TRUE(third->rb_contains(10)); +} + +TEST(PuffinFilesCacheClone, EmptyExcludedRowsReturnsNullptr) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 3, 100); + ASSERT_TRUE(key.has_value()); + + const auto files_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles); + const auto bytes_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return DataLakeObjectMetadata::ExcludedRowsPtr{}; + }; + + auto first = cache.getOrSetDeletionVector(*key, load_fn); + auto second = cache.getOrSetDeletionVector(*key, load_fn); + + EXPECT_EQ(first, nullptr); + EXPECT_EQ(second, nullptr); + EXPECT_EQ(load_calls, 1); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), files_before + 1); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), bytes_before + 1); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp new file mode 100644 index 000000000000..2168c5d16b70 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp @@ -0,0 +1,145 @@ +#include + +#include +#include + +using namespace DB; + +namespace +{ + +constexpr const char * kDefaultStorageIdentity = "Local:///test-prefix"; + +std::optional makeKey( + const String & referenced_data_file, + UInt64 expected_cardinality = 2, + UInt64 data_file_record_count = 100, + const String & storage_identity = kDefaultStorageIdentity) +{ + return PuffinFilesCache::tryCreateKey( + storage_identity, + "puffin.bin", + "etag-1", + 100, + 200, + referenced_data_file, + expected_cardinality, + data_file_record_count); +} + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheKey, SamePuffinSliceDifferentReferencedDataFile) +{ + const auto key1 = makeKey("data/file-a.parquet"); + const auto key2 = makeKey("data/file-b.parquet"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, SameReferencedDataFileProducesEqualKeys) +{ + const auto key1 = makeKey("data/file-a.parquet"); + const auto key2 = makeKey("data/file-a.parquet"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_EQ(*key1, *key2); + EXPECT_EQ(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentExpectedCardinalityProducesUnequalKeys) +{ + const auto key1 = makeKey("data/file-a.parquet", /*expected_cardinality=*/2); + const auto key2 = makeKey("data/file-a.parquet", /*expected_cardinality=*/3); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentDataFileRecordCountProducesUnequalKeys) +{ + const auto key1 = makeKey("data/file-a.parquet", /*expected_cardinality=*/2, /*data_file_record_count=*/100); + const auto key2 = makeKey("data/file-a.parquet", /*expected_cardinality=*/2, /*data_file_record_count=*/50); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentStorageIdentityProducesUnequalKeys) +{ + const auto key1 = makeKey("data/file-a.parquet", 2, 100, "S3://bucket-a/warehouse"); + const auto key2 = makeKey("data/file-a.parquet", 2, 100, "S3://bucket-b/warehouse"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, EmptyEtagReturnsNullopt) +{ + const auto key = PuffinFilesCache::tryCreateKey( + kDefaultStorageIdentity, "puffin.bin", "", 100, 200, "data/file-a.parquet", 2, 100); + EXPECT_FALSE(key.has_value()); +} + +TEST(PuffinFilesCacheKey, DifferentStorageIdentityDoesNotHitShare) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key_a = makeKey("data/file-a.parquet", 2, 100, "S3://bucket-a/warehouse"); + const auto key_b = makeKey("data/file-a.parquet", 2, 100, "S3://bucket-b/warehouse"); + ASSERT_TRUE(key_a.has_value()); + ASSERT_TRUE(key_b.has_value()); + + size_t load_a_calls = 0; + size_t load_b_calls = 0; + + auto first = cache.getOrSetDeletionVector(*key_a, [&]() + { + ++load_a_calls; + return makeExcludedRows({1, 2}); + }); + + auto second = cache.getOrSetDeletionVector(*key_b, [&]() + { + ++load_b_calls; + return makeExcludedRows({10, 20}); + }); + + ASSERT_EQ(load_a_calls, 1); + ASSERT_EQ(load_b_calls, 1); + ASSERT_TRUE(first); + ASSERT_TRUE(second); + EXPECT_TRUE(first->rb_contains(1)); + EXPECT_FALSE(first->rb_contains(10)); + EXPECT_TRUE(second->rb_contains(10)); + EXPECT_FALSE(second->rb_contains(1)); + + auto third = cache.getOrSetDeletionVector(*key_b, [&]() + { + ++load_b_calls; + return makeExcludedRows({99}); + }); + + ASSERT_EQ(load_b_calls, 1); + ASSERT_TRUE(third); + EXPECT_TRUE(third->rb_contains(10)); + EXPECT_FALSE(third->rb_contains(99)); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp new file mode 100644 index 000000000000..56f96b9aa65d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp @@ -0,0 +1,199 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheHits; +extern const Event PuffinFilesCacheMisses; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + auto & counters = CurrentThread::getProfileEvents(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; + + size_t load_calls = 0; + const auto result = cache.getOrSetDeletionVector( + *key, + [&]() + { + ++load_calls; + /// Simulate concurrent SYSTEM DROP while this key is loading: CacheBase then returns + /// `{value, false}` because the insert token was discarded. + cache.clear(); + return makeExcludedRows({1}); + }); + + ASSERT_EQ(load_calls, 1u); + ASSERT_NE(result, nullptr); + EXPECT_TRUE(result->rb_contains(1)); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 1u); + + /// Entry was not inserted after clear; the next lookup must load again. + const auto second = cache.getOrSetDeletionVector(*key, [&]() + { + ++load_calls; + return makeExcludedRows({1}); + }); + ASSERT_EQ(load_calls, 2u); + ASSERT_NE(second, nullptr); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 2u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); +} + +TEST(PuffinFilesCacheMetrics, WaiterOfClearDiscardedLoadCountsAsMiss) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-waiter", 100, 200, "data/file-w.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + const auto hits_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses]; + + std::promise load_started; + std::promise allow_load_finish; + std::promise waiter_entered; + auto load_started_future = load_started.get_future(); + auto allow_load_finish_future = allow_load_finish.get_future(); + auto waiter_entered_future = waiter_entered.get_future(); + + std::atomic load_calls{0}; + std::atomic waiter_load_called{false}; + + std::thread producer( + [&]() + { + cache.getOrSetDeletionVector( + *key, + [&]() + { + ++load_calls; + load_started.set_value(); + allow_load_finish_future.wait(); + cache.clear(); + return makeExcludedRows({42}); + }); + }); + + ASSERT_EQ(load_started_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + + std::thread waiter( + [&]() + { + waiter_entered.set_value(); + cache.getOrSetDeletionVector( + *key, + [&]() + { + waiter_load_called.store(true); + return makeExcludedRows({99}); + }); + }); + + ASSERT_EQ(waiter_entered_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + /// Give the waiter time to block on the shared insert token before the producer finishes. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + allow_load_finish.set_value(); + producer.join(); + waiter.join(); + + EXPECT_EQ(load_calls.load(), 1u); + EXPECT_FALSE(waiter_load_called.load()); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 2u); +} + +TEST(PuffinFilesCacheMetrics, OrdinaryHitAndMissCounters) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-2", 100, 200, "data/file-b.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + auto & counters = CurrentThread::getProfileEvents(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({7}); + }; + + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + + EXPECT_EQ(load_calls, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); +} + +TEST(PuffinFilesCacheMetrics, HitRemainsHitWhenCacheClearedAfterLookup) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-hit-clear", 100, 200, "data/file-c.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + auto & counters = CurrentThread::getProfileEvents(); + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({3}); + }; + + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; + + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + EXPECT_EQ(load_calls, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 0u); + + /// Clearing after the hit must not rewrite the already-recorded hit as a miss. The old + /// contains()-after-getOrSet path could race here with SYSTEM DROP PUFFIN FILES CACHE. + cache.clear(); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 0u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp new file mode 100644 index 000000000000..375d2f47d2a5 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp @@ -0,0 +1,95 @@ +#include + +#include +#include +#include +#include +#include + +using namespace DB; + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeLargeSparseExcludedRows(size_t keys = 33) +{ + auto excluded_rows = std::make_shared(); + for (size_t i = 0; i < keys; ++i) + excluded_rows->add((static_cast(i) + 1) << 32); + return excluded_rows; +} + +roaring::Roaring64Map makeLargeSparseRoaring64(size_t keys = 33) +{ + roaring::Roaring64Map bitmap; + for (size_t i = 0; i < keys; ++i) + bitmap.add((static_cast(i) + 1) << 32); + return bitmap; +} + +} + +TEST(RoaringBitmapWithSmallSetMemory, LargeSparseBitmapAllocatedBytesExceedCardinalityEstimate) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + + ASSERT_TRUE(excluded_rows->isLarge()); + EXPECT_GT(excluded_rows->getAllocatedBytes(), excluded_rows->size() * sizeof(size_t)); +} + +TEST(RoaringBitmapWithSmallSetMemory, LargeSparseBitmapAllocatedBytesExceedSerializedSize) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + const auto serialized = makeLargeSparseRoaring64().getSizeInBytes(/*portable=*/true); + + ASSERT_TRUE(excluded_rows->isLarge()); + EXPECT_GT(excluded_rows->getAllocatedBytes(), serialized); +} + +TEST(PuffinFilesCacheWeight, UsesRoaringAllocatedBytesForWeight) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 33, 1000); + ASSERT_TRUE(key.has_value()); + + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + cache.getOrSetDeletionVector(*key, [&]() { return excluded_rows; }); + + EXPECT_GE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), excluded_rows->getAllocatedBytes()); +} + +TEST(PuffinFilesCacheWeight, EvictsSparseBitmapsUnderSmallByteLimit) +{ + /// Each sparse entry weighs far more than cardinality × sizeof(size_t); a tiny byte + /// limit must not retain many of them if weight uses allocated bytes. + const auto entry_weight = makeLargeSparseExcludedRows(16)->getAllocatedBytes(); + ASSERT_GT(entry_weight, 0u); + + const size_t max_bytes = static_cast(entry_weight) + 200; // roughly one entry + overhead + PuffinFilesCache cache("SLRU", max_bytes, /*max_count=*/100, /*size_ratio=*/0.5); + + for (size_t i = 0; i < 8; ++i) + { + const auto key = PuffinFilesCache::tryCreateKey( + "Local:///test-prefix", + "puffin.bin", + "etag-" + std::to_string(i), + 100, + 200, + "data/file-" + std::to_string(i) + ".parquet", + 16, + 1000); + ASSERT_TRUE(key.has_value()); + cache.getOrSetDeletionVector(*key, [&]() { return makeLargeSparseExcludedRows(16); }); + } + + EXPECT_LE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), static_cast(max_bytes)); + EXPECT_LT(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), 8); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp new file mode 100644 index 000000000000..d94e7304b31b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp @@ -0,0 +1,52 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +TEST(PuffinNonSeekableBufferLimit, StopsBeforeExceedingAbsoluteLimit) +{ + constexpr size_t max_buffered_size = 100; + std::vector out = {'P', 'F', 'A', '1'}; + + /// Stream after the already-buffered header would push the total over the limit. + const String rest(max_buffered_size, 'x'); + ReadBufferFromString buf(rest); + + EXPECT_THROW(appendReadBufferWithAbsoluteSizeLimit(buf, out, max_buffered_size), Exception); + EXPECT_LE(out.size(), max_buffered_size); +} + +TEST(PuffinNonSeekableBufferLimit, AcceptsInputWithinLimit) +{ + constexpr size_t max_buffered_size = 100; + std::vector out = {'P', 'F', 'A', '1'}; + + const String rest(50, 'y'); + ReadBufferFromString buf(rest); + + ASSERT_NO_THROW(appendReadBufferWithAbsoluteSizeLimit(buf, out, max_buffered_size)); + EXPECT_EQ(out.size(), 54u); + EXPECT_EQ(out[0], 'P'); + EXPECT_EQ(out[4], 'y'); +} + +TEST(PuffinNonSeekableBufferLimit, RejectsAlreadyOversizedPrefix) +{ + std::vector out(101, 'z'); + ReadBufferFromString buf(String{}); + + EXPECT_THROW(appendReadBufferWithAbsoluteSizeLimit(buf, out, /*max_buffered_size=*/100), Exception); +} + +TEST(PuffinNonSeekableBufferLimit, ProductionCeilingCoversMaxDvPlusFooter) +{ + EXPECT_EQ( + PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE, + PUFFIN_MAGIC_SIZE + PUFFIN_DV_MAX_BLOB_SIZE + PUFFIN_MAGIC_SIZE + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE + + PUFFIN_FOOTER_TRAILER_SIZE); +} diff --git a/src/Storages/ObjectStorage/IObjectIterator.cpp b/src/Storages/ObjectStorage/IObjectIterator.cpp index 7ae3d1cbb1b2..b03ca71b325c 100644 --- a/src/Storages/ObjectStorage/IObjectIterator.cpp +++ b/src/Storages/ObjectStorage/IObjectIterator.cpp @@ -170,19 +170,19 @@ ObjectInfoPtr ObjectIteratorSplitByBuckets::next(size_t id) auto file_bucket_infos = splitter->splitToBuckets(bucket_size, *buffer, format_settings); for (const auto & file_bucket : file_bucket_infos) { - auto copy_object_info = *last_object_info; + auto copy_object_info = last_object_info->clone(); if (has_cache_entry) { auto filtered = file_bucket->filterByMatchingRowGroups(matching_row_groups); if (!filtered) continue; - copy_object_info.file_bucket_info = std::move(filtered); + copy_object_info->file_bucket_info = std::move(filtered); } else { - copy_object_info.file_bucket_info = file_bucket; + copy_object_info->file_bucket_info = file_bucket; } - pending_objects_info.push(std::make_shared(copy_object_info)); + pending_objects_info.push(std::move(copy_object_info)); } } } diff --git a/src/Storages/ObjectStorage/IObjectIterator.h b/src/Storages/ObjectStorage/IObjectIterator.h index 47febf269ee4..c922f0a70243 100644 --- a/src/Storages/ObjectStorage/IObjectIterator.h +++ b/src/Storages/ObjectStorage/IObjectIterator.h @@ -53,6 +53,10 @@ struct ObjectInfo FileBucketInfoPtr file_bucket_info; + /// Polymorphic copy. Used when splitting files into buckets so derived metadata + /// (e.g. Iceberg equality / position deletes) is not sliced away. + virtual ObjectInfoPtr clone() const { return std::make_shared(*this); } + String getIdentifier() const; }; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index d004243b3131..32bd77ddbe57 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -122,6 +122,39 @@ static void logIcebergFileStats(const ObjectInfoPtr & object_info, const LoggerP #endif } +/// Whether reading this object goes through row-level delete transformers (Iceberg +/// position/equality deletes, Delta Lake deletion vectors). The count-from-files cache +/// is keyed only by the file path and its modification time, but delete files change the +/// number of rows the file contributes WITHOUT touching the file itself, so both +/// directions are unsafe: a count cached before a delete resurfaces deleted rows, and a +/// count cached after it goes stale once the deletes are compacted away. Such files must +/// neither use nor populate the cache. +static bool hasAttachedDeletes(const ObjectInfo & object_info) +{ +#if USE_AVRO + if (const auto * iceberg_object = dynamic_cast(&object_info)) + { + if (!iceberg_object->info.position_deletes_objects.empty() || !iceberg_object->info.equality_deletes_objects.empty()) + return true; + } +#endif + return object_info.data_lake_metadata && object_info.data_lake_metadata->excluded_rows + && object_info.data_lake_metadata->excluded_rows->size() > 0; +} + +/// Count-from-files cache key is data-file identity only. Skip when row filtering can change +/// independently (DVs / selection vectors, Iceberg eq/pos deletes) or when the task is a bucket subset. +static bool canUseCountFromFilesCache(const ObjectInfoPtr & object_info) +{ + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata) || object_info->file_bucket_info) + return false; +#if USE_AVRO + if (hasIcebergEqualityDeletes(object_info) || hasIcebergPositionDeletes(object_info)) + return false; +#endif + return true; +} + StorageObjectStorageSource::StorageObjectStorageSource( const StorageID & storage_id_, String name_, @@ -585,7 +618,8 @@ Chunk StorageObjectStorageSource::generate() } if (reader.getInputFormat() && read_context->getSettingsRef()[Setting::use_cache_for_count_from_files] - && !format_filter_info->filter_actions_dag) + && !format_filter_info->filter_actions_dag + && canUseCountFromFilesCache(reader.getObjectInfo())) addNumRowsToCache(*reader.getObjectInfo(), total_rows_in_file); total_rows_in_file = 0; @@ -747,8 +781,35 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade return schema_cache->tryGetNumRows(cache_key, get_last_mod_time); }; - std::optional num_rows_from_cache - = need_only_count && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] ? try_get_num_rows_from_cache() : std::nullopt; + /// Row-level delete transformers need real row values: an equality-delete FilterTransform + /// evaluates its predicate against column values, but the count-only fast path + /// (`input_format->needOnlyCount()`) makes the format emit synthetic chunks filled with + /// default values, so the predicate would filter the wrong rows and count() would come + /// back wrong. Position deletes and deletion vectors filter by row index, which synthetic + /// chunks do preserve, but they would still build the huge synthetic chunks only to drop + /// rows from them, so the fast path is disabled for any attached deletes. This also + /// covers the count-from-cache shortcut below: a cached per-file row count is keyed only + /// by path + mtime, both untouched by delete files, so it must not be used either. + need_only_count = need_only_count && !hasAttachedDeletes(*object_info); + + /// The count-from-cache shortcut builds a `ConstChunkGenerator` without opening the read buffer, so a + /// requested `_headers` virtual column (the HTTP response headers of the data `GET`) would have to fall + /// back to the metadata-probe headers (usually a `HEAD`), which can differ from the actual `GET` + /// response. Skip the shortcut when `_headers` is requested so the real `GET` headers are used. + const bool headers_requested = read_from_format_info.requested_virtual_columns.contains("_headers"); + +#if USE_AVRO + /// Equality deletes filter by column values; need_only_count emits default-filled chunks. + const bool effective_need_only_count = need_only_count && !hasIcebergEqualityDeletes(object_info); +#else + const bool effective_need_only_count = need_only_count; +#endif + + const bool can_use_count_cache = effective_need_only_count && !headers_requested + && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] + && canUseCountFromFilesCache(object_info); + + std::optional num_rows_from_cache = can_use_count_cache ? try_get_num_rows_from_cache() : std::nullopt; if (num_rows_from_cache) { @@ -919,7 +980,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade filter_info, true /* is_remote_fs */, compression_method, - need_only_count, + effective_need_only_count, std::nullopt /*min_block_size_bytes*/, std::nullopt /*min_block_size_rows*/, std::nullopt /*max_block_size_bytes*/); @@ -937,22 +998,22 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade filter_info, true /* is_remote_fs */, compression_method, - need_only_count); + effective_need_only_count); } input_format->setBucketsToRead(object_info->file_bucket_info); input_format->setSerializationHints(read_from_format_info.serialization_hints); - if (need_only_count) + if (effective_need_only_count) input_format->needOnlyCount(); builder.init(Pipe(input_format)); - configuration->addDeleteTransformers(object_info, builder, format_settings, parser_shared_resources, context_); - - if (object_info->data_lake_metadata - && object_info->data_lake_metadata->excluded_rows - && object_info->data_lake_metadata->excluded_rows->size() > 0) + /// Deletion vectors (and selection vectors) address absolute file row numbers via + /// `ChunkInfoRowNumbers`. Iceberg equality deletes use a plain `FilterTransform` that + /// shrinks the chunk without maintaining `applied_filter`, so DV must run first — + /// otherwise later DV filtering maps dense post-equality indices to the wrong file rows. + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata)) { builder.addSimpleTransform([&](const SharedHeader & header) { @@ -960,6 +1021,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade }); } + configuration->addDeleteTransformers(object_info, builder, format_settings, parser_shared_resources, context_); + std::optional schema_transform; if (object_info->data_lake_metadata && object_info->data_lake_metadata->schema_transform) { diff --git a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp index b00c1d609fa1..c0112b4746c9 100644 --- a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp +++ b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp @@ -1,9 +1,29 @@ #include +#include +#include +#include +#include +#include +#include #include #include +#include + +#if USE_PARQUET +#include +#include +#endif using namespace DB; +namespace DB +{ +namespace ErrorCodes +{ +extern const int UNKNOWN_PROTOCOL; +} +} + namespace { class TestIterator : public IObjectIterator @@ -14,6 +34,11 @@ namespace for (const auto & path : paths) objects.push_back(std::make_shared(path)); } + explicit TestIterator(const std::vector & paths) + { + for (const auto & path : paths) + objects.push_back(std::make_shared(path)); + } ~TestIterator() override = default; ObjectInfoPtr next(size_t) override { @@ -61,6 +86,22 @@ namespace return true; } + bool extractNForReplica( + StorageObjectStorageStableTaskDistributor & distributor, + std::vector> & read_source_indices, + size_t replica_id, + size_t n) + { + for (size_t i = 0; i < n; ++i) + { + ObjectInfoPtr object = distributor.getNextTask(replica_id); + if (!object) + return false; + read_source_indices.emplace_back(object->relative_path_with_metadata.read_source_index); + } + return true; + } + // Head of the list must contains all paths for files with numbers from file_nums bool checkHead(const std::vector & paths, std::vector file_nums) { @@ -231,3 +272,285 @@ TEST(RendezvousHashing, MultipleNodesReducedClusterOneByOne) ASSERT_TRUE(checkHead(paths0, {1, 5, 6, 7, 8, 9})); ASSERT_TRUE(checkHead(paths1, {0, 2, 3, 4})); } + +TEST(RendezvousHashing, DoesNotDeduplicateSamePathFromDifferentReadSources) +{ + const auto path = makePath(0); + std::vector paths; + paths.emplace_back(path, 0); + paths.emplace_back(path, 1); + + auto iterator = std::make_shared(paths); + std::vector replicas = {"replica0", "replica1", "replica2", "replica3"}; + StorageObjectStorageStableTaskDistributor distributor(iterator, std::move(replicas), false); + + std::vector> read_source_indices; + ASSERT_TRUE(extractNForReplica(distributor, read_source_indices, 0, 2)); + + std::set actual; + for (const auto & read_source_index : read_source_indices) + { + ASSERT_TRUE(read_source_index.has_value()); + actual.insert(*read_source_index); + } + + ASSERT_EQ(actual, (std::set{0, 1})); +} + +TEST(RelativePathWithMetadata, FileNameKeepsObjectKeyByDefault) +{ + RelativePathWithMetadata object_path("dir/part?.parquet#fragment"); + ASSERT_EQ(object_path.getFileName(), "part?.parquet#fragment"); + + RelativePathWithMetadata web_path("dir/part.tsv.gz?download=1#fragment"); + web_path.derive_file_name_from_url_path = true; + ASSERT_EQ(web_path.getFileName(), "part.tsv.gz"); +} + +TEST(ObjectInfo, IdentifierWithoutFileBucketInfoKeepsReadSourceIndex) +{ + ObjectInfo object_info(RelativePathWithMetadata("dir/file.parquet", 7)); + ASSERT_EQ(object_info.getIdentifier(/*include_file_bucket_info=*/ false), "7:dir/file.parquet"); +} + +TEST(ClusterFunctionReadTaskResponse, PreservesReadSourceIndex) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file"; + response.read_source_index = 42; + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + auto object_info = deserialized.getObjectInfo(); + ASSERT_TRUE(object_info->relative_path_with_metadata.read_source_index.has_value()); + ASSERT_EQ(*object_info->relative_path_with_metadata.read_source_index, 42); +} + +TEST(ClusterFunctionReadTaskResponse, RejectsNonEmptyExcludedRowsOnOldProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + response.data_lake_metadata.excluded_rows->add(7); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("excluded_rows"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsEmptyExcludedRowsOnOldProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsExcludedRowsOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + response.data_lake_metadata.excluded_rows->add(3); + response.data_lake_metadata.excluded_rows->add(9); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.data_lake_metadata.excluded_rows); + EXPECT_EQ(deserialized.data_lake_metadata.excluded_rows->size(), 2u); + EXPECT_TRUE(deserialized.data_lake_metadata.excluded_rows->rb_contains(3)); + EXPECT_TRUE(deserialized.data_lake_metadata.excluded_rows->rb_contains(9)); +} + +TEST(ClusterFunctionReadTaskResponse, RejectsEqualityDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "/path/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1}, + .schema_id = 0, + }); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, RejectsPositionDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "/path/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 1, + }); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsIcebergInfoWithoutDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsIcebergDeletesOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->data_object_file_path_key + = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/path/file.parquet"); + response.iceberg_info->file_format = "PARQUET"; + response.iceberg_info->equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "/path/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1, 2}, + .schema_id = 7, + }); + response.iceberg_info->position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "/path/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 42, + }); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.iceberg_info.has_value()); + ASSERT_EQ(deserialized.iceberg_info->equality_deletes_objects.size(), 1u); + EXPECT_EQ(deserialized.iceberg_info->equality_deletes_objects[0].file_path, "/path/eq.parquet"); + ASSERT_EQ(deserialized.iceberg_info->position_deletes_objects.size(), 1u); + EXPECT_EQ(deserialized.iceberg_info->position_deletes_objects[0].file_path, "/path/pos.parquet"); +} + +#if USE_PARQUET + +TEST(ClusterFunctionReadTaskResponse, RejectsFileBucketInfoOnProtocolBeforeFileBuckets) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.file_bucket_info = std::make_shared(std::vector{0, 1}); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("file_bucket_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsMissingFileBucketInfoOnProtocolBeforeFileBuckets) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsFileBucketInfoOnSupportedProtocol) +{ + tryRegisterFormats(); + + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.file_bucket_info = std::make_shared(std::vector{2, 5}); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.file_bucket_info); + auto * parquet_buckets = dynamic_cast(deserialized.file_bucket_info.get()); + ASSERT_TRUE(parquet_buckets); + EXPECT_EQ(parquet_buckets->row_group_ids, (std::vector{2, 5})); +} + +#endif diff --git a/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py new file mode 100644 index 000000000000..a13f7dfaf234 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py @@ -0,0 +1,424 @@ +import uuid + +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_uuid_str, + get_creation_expression, +) + + +def get_array(query_result: str): + return sorted([int(x) for x in query_result.strip().split("\n") if x]) + + +def upload_table(cluster, storage_type, table_name): + default_upload_directory( + cluster, + storage_type, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + assert int(instance.query(f"SELECT count() FROM {expression}")) == 200 - len(deleted_ids) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == [ + x for x in range(200) if x not in deleted_ids + ] + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_aggregates(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): + """Aggregates over Iceberg v3 tables must ignore rows covered by deletion vectors.""" + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_aggregates_" + storage_type + "_" + get_uuid_str() + deleted_ids = {2, 5, 7, 50, 99} + remaining_ids = [i for i in range(100) if i not in deleted_ids] + # value = 10 * id, so sum/avg expectations stay integer-friendly where possible. + remaining_values = [10 * i for i in remaining_ids] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, value bigint, group_id int) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, 10 * id, CAST(id % 3 AS INT) + FROM range(0, 100) + """ + ) + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in sorted(deleted_ids))})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + spark_row = spark.sql( + f""" + SELECT + count(*) AS cnt, + sum(id) AS sum_id, + sum(value) AS sum_value, + min(id) AS min_id, + max(id) AS max_id, + avg(value) AS avg_value + FROM {table_name} + """ + ).collect()[0] + + expected_count = len(remaining_ids) + expected_sum_id = sum(remaining_ids) + expected_sum_value = sum(remaining_values) + expected_min_id = min(remaining_ids) + expected_max_id = max(remaining_ids) + expected_avg_value = expected_sum_value / expected_count + + assert spark_row["cnt"] == expected_count + assert spark_row["sum_id"] == expected_sum_id + assert spark_row["sum_value"] == expected_sum_value + assert spark_row["min_id"] == expected_min_id + assert spark_row["max_id"] == expected_max_id + assert abs(float(spark_row["avg_value"]) - expected_avg_value) < 1e-9 + + ch_row = instance.query( + f""" + SELECT + count(), + count(id), + sum(id), + sum(value), + min(id), + max(id), + avg(value), + uniqExact(id), + countIf(id % 2 = 0), + sumIf(value, id % 2 = 0) + FROM {expression} + """ + ).strip().split("\t") + + assert int(ch_row[0]) == expected_count + assert int(ch_row[1]) == expected_count + assert int(ch_row[2]) == expected_sum_id + assert int(ch_row[3]) == expected_sum_value + assert int(ch_row[4]) == expected_min_id + assert int(ch_row[5]) == expected_max_id + assert abs(float(ch_row[6]) - expected_avg_value) < 1e-9 + assert int(ch_row[7]) == expected_count + + expected_even_ids = [i for i in remaining_ids if i % 2 == 0] + expected_count_if = len(expected_even_ids) + expected_sum_if = sum(10 * i for i in expected_even_ids) + assert int(ch_row[8]) == expected_count_if + assert int(ch_row[9]) == expected_sum_if + + # Trivial COUNT must match the full scan once deletion vectors are applied. + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={"optimize_trivial_count_query": 1}, + ) + ) + == expected_count + ) + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={"optimize_trivial_count_query": 0}, + ) + ) + == expected_count + ) + + # GROUP BY aggregates must also exclude deleted rows. + spark_groups = { + int(row["group_id"]): (int(row["cnt"]), int(row["sum_value"])) + for row in spark.sql( + f""" + SELECT group_id, count(*) AS cnt, sum(value) AS sum_value + FROM {table_name} + GROUP BY group_id + ORDER BY group_id + """ + ).collect() + } + ch_groups = {} + for line in instance.query( + f""" + SELECT group_id, count(), sum(value) + FROM {expression} + GROUP BY group_id + ORDER BY group_id + """ + ).strip().split("\n"): + group_id, cnt, sum_value = line.split("\t") + ch_groups[int(group_id)] = (int(cnt), int(sum_value)) + + expected_groups = {} + for group_id in (0, 1, 2): + ids = [i for i in remaining_ids if i % 3 == group_id] + expected_groups[group_id] = (len(ids), sum(10 * i for i in ids)) + + assert spark_groups == expected_groups + assert ch_groups == expected_groups + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_complex(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_complex_" + storage_type + "_" + get_uuid_str() + + def expected_complex_ids(): + ids = list(range(20, 90)) + list(range(100, 150)) + ids += [x for x in range(200, 250) if x not in {205, 210, 220}] + return sorted(ids) + + expected_ids = expected_complex_ids() + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, data string) USING iceberg + PARTITIONED BY (bucket(5, id)) + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f"INSERT INTO {table_name} SELECT id, char(id + ascii('a')) FROM range(10, 100)" + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + + assert int(instance.query(f"SELECT count(id) FROM {expression}")) == 90 + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(10, 100)) + + spark.sql(f"DELETE FROM {table_name} WHERE id < 20") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 100)) + + spark.sql(f"DELETE FROM {table_name} WHERE id >= 90") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + + spark.sql( + f"INSERT INTO {table_name} SELECT id, char(id + ascii('a')) FROM range(100, 200)" + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 200) + ) + + spark.sql(f"DELETE FROM {table_name} WHERE id >= 150") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 150) + ) + + spark.sql(f"ALTER TABLE {table_name} ADD COLUMNS (label string)") + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, char(id + ascii('a')), 'new' + FROM range(200, 250) + """ + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 150) + ) + list(range(200, 250)) + assert int(instance.query(f"SELECT count(id) FROM {expression} WHERE label = 'new'")) == 50 + + spark.sql(f"DELETE FROM {table_name} WHERE id IN (205, 210, 220)") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected_ids + assert int(instance.query(f"SELECT count(id) FROM {expression}")) == len(expected_ids) + + spark.sql(f"UPDATE {table_name} SET label = 'updated' WHERE id = 25") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert instance.query(f"SELECT label FROM {expression} WHERE id = 25").strip() == "updated" + assert int(instance.query(f"SELECT count(id) FROM {expression} WHERE label = 'updated'")) == 1 + + spark.sql(f"CALL system.rewrite_data_files(table => '{table_name}')") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected_ids + + assert get_array( + instance.query( + f"SELECT id FROM {expression} WHERE id % 3 = 0" + ) + ) == sorted([x for x in expected_ids if x % 3 == 0]) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_deletion_vectors_puffin_files_cache(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_cache_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + + instance.query("SYSTEM DROP PUFFIN FILES CACHE") + + query_id1 = f"{table_name}-{uuid.uuid4()}" + query_id2 = f"{table_name}-{uuid.uuid4()}" + query_id3 = f"{table_name}-{uuid.uuid4()}" + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id1, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id2, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + instance.query("SYSTEM FLUSH LOGS") + + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id1}' AND type = 'QueryFinish'" + ) + ) > 0 + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheHits'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) > 0 + + puffin_reads_first = int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesRead'] FROM system.query_log WHERE query_id = '{query_id1}' AND type = 'QueryFinish'" + ) + ) + puffin_reads_second = int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesRead'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) + assert puffin_reads_first > 0 + assert puffin_reads_second == 0 + + instance.query("SYSTEM DROP PUFFIN FILES CACHE") + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id3, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + instance.query("SYSTEM FLUSH LOGS") + + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id3}' AND type = 'QueryFinish'" + ) + ) > int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index cc318eca54d0..467b53f3fb1d 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -128,6 +128,7 @@ SYSTEM DROP MARK CACHE ['SYSTEM CLEAR MARK CACHE','SYSTEM DROP MARK','DROP MARK SYSTEM DROP ICEBERG METADATA CACHE ['SYSTEM CLEAR ICEBERG_METADATA_CACHE','SYSTEM DROP ICEBERG_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP AVRO SCHEMA CACHE ['SYSTEM CLEAR AVRO SCHEMA CACHE','SYSTEM DROP AVRO SCHEMA CACHE','DROP AVRO SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PARQUET METADATA CACHE ['SYSTEM DROP PARQUET_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP PUFFIN FILES CACHE ['SYSTEM DROP PUFFIN FILES CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM PREWARM PRIMARY INDEX CACHE ['SYSTEM PREWARM PRIMARY INDEX','PREWARM PRIMARY INDEX CACHE','PREWARM PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PRIMARY INDEX CACHE ['SYSTEM CLEAR PRIMARY INDEX CACHE','SYSTEM DROP PRIMARY INDEX','DROP PRIMARY INDEX CACHE','DROP PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM CLEAR UNCOMPRESSED CACHE','SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.reference b/tests/queries/0_stateless/04117_parser_system_query_variants.reference index 2ffe7e9fb91f..3b81cbe7ae02 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.reference +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.reference @@ -15,6 +15,8 @@ SYSTEM CLEAR QUERY CONDITION CACHE SYSTEM CLEAR COMPILED EXPRESSION CACHE SYSTEM CLEAR ICEBERG METADATA CACHE SYSTEM CLEAR PARQUET METADATA CACHE +SYSTEM CLEAR PUFFIN FILES CACHE +SYSTEM CLEAR PUFFIN FILES CACHE SYSTEM CLEAR FILESYSTEM CACHE SYSTEM CLEAR DISTRIBUTED CACHE SYSTEM CLEAR DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.sql b/tests/queries/0_stateless/04117_parser_system_query_variants.sql index c48bb44537f9..bb68fdee5aa5 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.sql +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.sql @@ -25,6 +25,8 @@ EXPLAIN SYNTAX SYSTEM DROP QUERY CONDITION CACHE; EXPLAIN SYNTAX SYSTEM DROP COMPILED EXPRESSION CACHE; EXPLAIN SYNTAX SYSTEM DROP ICEBERG METADATA CACHE; EXPLAIN SYNTAX SYSTEM DROP PARQUET METADATA CACHE; +EXPLAIN SYNTAX SYSTEM DROP PUFFIN FILES CACHE; +EXPLAIN SYNTAX SYSTEM DROP PUFFIN_FILES_CACHE; EXPLAIN SYNTAX SYSTEM DROP FILESYSTEM CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster; diff --git a/tests/queries/0_stateless/04261_iceberg_deletion_vector.reference b/tests/queries/0_stateless/04261_iceberg_deletion_vector.reference new file mode 100644 index 000000000000..fe7ad97c7e35 --- /dev/null +++ b/tests/queries/0_stateless/04261_iceberg_deletion_vector.reference @@ -0,0 +1,197 @@ +196 +0 +1 +3 +4 +6 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 diff --git a/tests/queries/0_stateless/04261_iceberg_deletion_vector.sh b/tests/queries/0_stateless/04261_iceberg_deletion_vector.sh new file mode 100755 index 000000000000..b09a64d3e88d --- /dev/null +++ b/tests/queries/0_stateless/04261_iceberg_deletion_vector.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_PATH="${CURDIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_source" + +$CLICKHOUSE_LOCAL -q " +SELECT count() FROM icebergLocal('${TABLE_PATH}') +" + +$CLICKHOUSE_LOCAL -q " +SELECT id FROM icebergLocal('${TABLE_PATH}') ORDER BY id +" diff --git a/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference new file mode 100644 index 000000000000..32bd071ad3c5 --- /dev/null +++ b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference @@ -0,0 +1,172 @@ +167 +47 +1 +updated +119 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +200 +201 +202 +203 +204 +206 +207 +208 +209 +211 +212 +213 +214 +215 +216 +217 +218 +219 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 diff --git a/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh new file mode 100755 index 000000000000..0fe7da881e31 --- /dev/null +++ b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_PATH="${CURDIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_complex" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') +" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') WHERE label = 'new' +" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') WHERE label = 'updated' +" + +$CLICKHOUSE_LOCAL -q " +SELECT label FROM icebergLocal('${TABLE_PATH}') WHERE id = 25 +" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') WHERE label IS NULL +" + +$CLICKHOUSE_LOCAL -q " +SELECT id FROM icebergLocal('${TABLE_PATH}') ORDER BY id +" diff --git a/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference new file mode 100644 index 000000000000..50a04fbf6628 --- /dev/null +++ b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference @@ -0,0 +1,19 @@ +196 +PuffinFilesCacheMisses 1 +PuffinFilesRead 2 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 1 +PuffinFilesRead 2 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 2 +PuffinFilesRead 4 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 2 +PuffinFilesRead 6 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 2 +PuffinFilesRead 8 diff --git a/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh new file mode 100755 index 000000000000..f9565b8cf146 --- /dev/null +++ b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas, no-random-settings +# no-fasttest: depends on local iceberg fixture +# no-parallel: cache is system-wide and tests can affect each other in unexpected way +# no-parallel-replicas: profile events are not available on the second replica +# no-random-settings: we need to test the interaction of specific setting combinations + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_PATH="${CURDIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_source" + +$CLICKHOUSE_LOCAL -q " +SYSTEM DROP PUFFIN FILES CACHE; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 1; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 1; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SYSTEM DROP PUFFIN FILES CACHE; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 1; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SYSTEM DROP PUFFIN FILES CACHE; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 0; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 0; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; +" diff --git a/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh b/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh index 238205a43c66..8ae4bfff924e 100755 --- a/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh +++ b/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh @@ -23,6 +23,8 @@ SETTINGS input_format_allow_seeks = 0 " # Non-seekable path must reject a wrong leading magic before buffering the rest of the stream. +# Oversized valid-magic streams are covered by gtest_puffin_non_seekable_buffer_limit (absolute +# buffer ceiling is ~2 GiB + footer; too large for a CI fixture). echo "--- non-Puffin magic allow_seeks=0 ---" NOT_PUFFIN="${CLICKHOUSE_TMP}/04549_not_puffin.bin" # Prefix is wrong; trailing bytes would only matter if we buffered first then validated. diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md new file mode 100644 index 000000000000..f4cf3a4c173b --- /dev/null +++ b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md @@ -0,0 +1,35 @@ +# Iceberg v3 deletion vector fixtures + +Generated with `generate_iceberg_dv_fixture.py` (Spark + Iceberg 1.9.0). + +Warehouse: `dv_puffin_warehouse/` + +## `default/dv_puffin_source` + +Simple table with column `id BIGINT`: + +- 200 rows (`id` from 0 to 199) +- Deleted rows via Puffin deletion vector: 2, 5, 7, 100 + +## `default/dv_puffin_complex` + +Table with columns `id BIGINT`, `data STRING`, `label STRING` and multiple snapshots: + +- Initial insert `id` 10-99, then deletes `id < 20` and `id >= 90` +- Insert `id` 100-199, delete `id >= 150` +- Schema evolution (`label` column), insert `id` 200-249 with `label = 'new'` +- Delete `id` in (205, 210, 220), update `id = 25` to `label = 'updated'` +- Spark `rewrite_data_files` compaction + +Regenerate: + +```bash +python3 generate_iceberg_dv_fixture.py +``` + +Query paths for tests: + +```text +data_minio/dv_puffin_warehouse/default/dv_puffin_source +data_minio/dv_puffin_warehouse/default/dv_puffin_complex +``` diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..956042e717a548bc296df89f69e032150c44dd42 GIT binary patch literal 2141 zcmdUw&x+JA9LFmjW>=gy5FBz+nAoOG|BWoT2&>-q;z?x4pI>*vcG_-|jxNJqeGK2l zClJ|_2)@Cdd;x!55%e&7n8U#QQc9bA^UE(ELcXUb$1Dhf$KS3?eGfnW#`7o$@c#f` z2iQ)6U$1azlWdH#2f;Y_{`t$z9kL&c+)m?1r?agp2cFFMl+@6i`30FJ<&<=`?ZA2in08GpRA7gxnrh(% ztxP=>6E4~%ghn*JZCoeAm$gynuiM^1=)rjx;(8WSgwfo2;jwh*!Xd$YX&TdpdNuF* zsxr;|gRro;ov5J_eeM4lQ|+>=;p6SZ(CE4B)j9a1D2#@nj8R^eS|@o)6CRh8YgN#a zr#UTUq*IZJ3`$8OI>^&!ov>JzM{G)1Z5mM^Qw>?6UegVocXfQD0SO#w&X z00@bSo+HrWywyt30NMhyBNyZG%y|6C-*^1vh$cybK7HP*ThY0)HcOJL{k=o*`^EbY z(fMQM?d0RzFZJJK-buc``L;Wb6YP-~t?;OQzDMC~2Gk-2BLr;?-hm9(;T=UO>NSjN zP0J~(SyNQ3e32q=`0NK8uvVzWGzQ@10!9Kh%Ot>E25sMa2&iQ#8o``|A-Z{rW@l$` zK|^f2M-F;$YpKGD2abC^uEUVQPCzs_!5Tjc@_5wp^r>AukN`e-67O@d1<5)ec!;BW zPu2$Aq>Z*j?YepCdu_T`+`(`tRM7Kf3_qPoJIy1!J`@tII~dj4?cE{+ur4Igzli;tn`{5<^!I9E3f z==@qhv;Pfr)7&(m*|mT!ehYNefnTXC9YiK7EY?4NG;=w?=9z@32Pc~)vKXX}a-uf( uBkd*~HnkGG7p!OiC{&HgGFA%^Qj|@dSEZ^|6BnlsB7NT7z0}E7yM6$>E~FIz literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..ce4a05ab2431ee61dba5b610026f085560a79817 GIT binary patch literal 539 zcmY*WOK#jS40U>f9$}zW1B@!$kw0Ugi=aT!?y@feLmw&@VoQo7Pa6z!hhC}&C<2L4Krlx9X*1|KGI`F_QY72^#-de-tqpT<<0e;Bqu_ee5O{DfmWD+yA_-Q>efX< z{Fn5`B*f|X;+A|v&b*w$cNth1i!;}}+BtQ)_H2-8!c1|!xvzxq>t2{U|u zDr8oVa#gpGKa{N8jB;wAH|C%P<#p3i-UvYtpexGzx-2r4n2VmRA5NtSW`=R0o-fRqwxR$ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..7df1cec9ff2a84f8a891ed255c8a40b01403463d GIT binary patch literal 1351 zcmdUv&2AGh5XapD2QEP21tNnGNHO~n?`9Lkp+e&0f+}$Xq2P~kV|BCM+TK7F^#OPU zUV=y91vnutJPxyokOERJJyfc-hw=D7#0(D=KiDIuHv9=U_g zyW}4E`0o4*nh$qt_t#oNZjsyM>&x#yP<{V@_0vt}n`9qPuaN_CgS`Fy=HLhmSy5R# z8R+!=ipH}R=#*-xAzIh4C8+2;TV`}fjfGnGvOMEsUX4m#d>GQuh&Ie!WLDED%g_O* z9W)A9(=8)Pn)vhiaxp6wKJVFD9A01teB9 z(KVr!$@AK(lUJ?_kVFVEBzT|46iGS@QA8{~grrX}TiC|BL@#G;XN+xTpM;0`I#7cV zT^+a1lv}hld|n48R?lRoP9Q$ZlC)opGXs?|6D=6ei<0p|2sQzoF<#|q%6XBN6T?#0 ztFtGq@K{wuYD#!*8&TVMhX0b&^Yh^)V6L_}msf!^zX<2iPR^st;XK)X&!elrIll;J zzLRs7B50ff|2DGnfPJW$?4SH$E7TkYuNB}lI39G^!=Nk<6Fqo@O}DUdFxFBS$ww7H i24$tOpT-iTl0`Mn%R-NJg`dbC<7MBj;{e0ePp&^TI-~Ib literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fb364a125c2fa75b550e84d7a49497b2ed3e9d01 GIT binary patch literal 779 zcmZ`&&ubGw6n?Y2>K=k9bcS76N*J1u!Zv18^OGFvO?oP#Ac)}pN?l0OWU~>q1d8O~ zNvsMAVntCY9yFkW7fmDw@1A<~=FxvZ@J-eLguWv5_3r2i`KM`3f=HH-GufTuy~lgcUcP<*afDO=0kVJv0gD2-fF%JLRgu=l z*PZKja zY{H6vCIIY_LKoRLndX2E6~@yl+mRPUt#&+cea8{D#-%)l1u4oLDAr*-cT8JClcgrx%1iCXnr$vtEs@)+XMWCE&U!^bH9FSpr?QC0#e|{%YBa9z=_jA_}FVkYs0TV7qOzSw-63 zil-jLgU8~@b9?Y2_y-8yDtOt0-g*&xP+AXyZxXky6fuO4nK$3}-uLDu^VjaU2)K%K zxc6;$?}CL{+UlqS6y99@1q*$ESNre@zQ6%|g(LV0=`(y+0hKWTqTj%W`0n%jw-?W; z4F*JkvV^*X>k@8Bn3J#|VUYkeDxAlo{_*!r0~OQ;_3HpCaXR9OWJ#Wd$3G5tRar|r z>+}-QR3e#|71Pl7X(u0A4lJW$axi(8-ju}3rok4^g;gqM8WR6Hvbc^LT&A@67Q<~u z>m-%M`wGq`yKx}N?VCff=eGq8lig7;l(2bE0nE@+YkILa3Kh+=C{Yp`YJ0wDQ*1bi zkv+?7KP9tZ@K*=B<|1%fh&xX+AI~rDjA1geY*ZGb*A}`&@hJo-!v{dW{ zkHoTPd|>N#{B%{=Gj=zQdO>VEc6u}sHtm^`W4A)lZg%aN`|GxhxQ^ZNmL;Edjva}$ zUKj~FAhmD?x!Nn8C419JC6gx-Ik|s}$Ut~THgzIPru%>0gH14ISG{PhC*q0aDEC-I z-EgIobBk`J=;jK3FKjn+L2a^9^Xo0I)v8pdr-eJ^`$4VlR%_GEpfuGg*8|rrO?qQy QgTAlyg%=9IJpJas0pwQ6<^TWy literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6dfde23d8eaeb51ba7d8cb37aefe5efe95801946 GIT binary patch literal 969 zcma)5&1(}u6n{HS(;zV*^bI?*hcM8PLK?G4)3hNM!9yuM1VJcR5Bo7`wrtXLvnklp zs|bQ8{{TVo;K7@>o(ooiLGS3%00_acjGtco{*zrG983uWg!3_4ViT{vy!?J^ z0sVZ6ULo{o;98+_i6ibXp=YKk=S*aRgWyaQsH_;JoeNz%UyQFz)Pz2@Zek3cL7P)n zVen%-B#UGt=UT&V?B#$M`@74bW%~-#fT=a6GtYg*0JCJKs9I|c>M6q@nK-oUTek8k zmX2dory*JeFfb+Rkj{vRj#>ChtC_&X;=I*HV?6yD`O8CGscjoXQ}+Pqu1)>_Fh6N7}O z15RS|NpiC!uTyC7cT2bn{F}yEA;aIY)#w29oaZQ81xMwf(^Wl7|Eyie@aaIwrW}UB z$O&ahj<+^ES2imp+3}R`4rTN4g^VI*N%pN?geR{g19dv`0!14g%O9%JRLno=AIZzo zm{UzS(Lw%$MJ0H1yjsGGPy0WJPcA{<8(6{VNQE1{Ab+NUp?A{Hn`N_JHuFV$MHZFK}Cd+2u2Y_vpcgjHEy<-By=gG z0;7xc)Wt3=Xwe-kDuR$K>%oXF4-o}DL=gq~AW=}%nO#?fAR2}_|KdL%L?Ipj-x2;#2&p!{0eX{P2^aZK*vnH|?*vItOHVS$R+WkTCP&nptljm0liLFU>1J;CV&E8t+e5dDWvnQ;=g_GAQ1eH@0>p|KM68N_Pz$@58c|?}Zv36#C?ReXlWqm19JPBNRM ziy<(4ise-Y?VQj?K?oBQGE8I;&4%F zz#}ds;a5%+0X*o&s^5*DJ7(L$3p31^5jB#VD^sy8Myn<~WjaWDSDQf^NGhGo#L`42 zg;a&9lZGmp#7)D}Q>0;i2eBiHOcGkFy`OqANg6vcX40UExMrmcx!BCwp4d#Xa-k+T zQDYz9KdLAMb4p$HL|y)j|D$k27KBx8T5?CmNLREbeLIb0%G{ptDV0j7Qt<_%8Pn2z zvASTWE*g$&@p!1VzTQx(qtRGhSgEb6*JD-H@tSZUDzwJ0z^n z4nBYUPBfOrSY|Rn{{5w&aLXmIMCU*O70$tVxBwR+4?0w#0ZY(?7Tkn*rIslq2mwI+ zTf&|Yw)OO2eEOe;9|$Z{EQ3*G)+6(PqS+fieDwI)^Osw%-n^T}JEtUi*PGb#=_2sfhJN7T2BJOvs8?sTw<0$67*aZhM5Fva z2nF~N?9L`A&At_^4ZU!o8)WW#!N6bbWsHJRE*P1-J@h+n#;KIb6}#HD+U>Ggt9izv kZ9A2!VODCcQ(SCct~!QMELroaju#i*-4A?1EqwC70NR(C!2kdN literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..43f7e9c338c66ca5c0cccdcd8aaf0090520bf278 GIT binary patch literal 766 zcmZ`%&ubGw6n?Y2>K;N#r8Dfx62d}40^7KonxEvL;6ZvSq9BOi{+P5YNt$jpf|MSM zU=LnO@gzA(LBVAe4W{VRjo@W zV%O_%ssL4Tph;nHRRWqqR|(wM%|S&m41^+wp<8({sG{dUMV2t2DVAX)iH%5JZB0c@ zl`<$COPxkh2F?uGp;b(z13Q!{W1l6u2!bM&P)rm|NInpDK=Qk!RfPgxPb29+nlStK z1c3dNX@F|82{evtWB^Va4co4P3{F_4&yD=H!#l1K(mq+{4j5`*_&M=}%W0%6E2$!v+x`5+FtWHue6ec`SI zi0Qu;uNkIi(-!pA1|sK{Z*o8r81iMKX0$<;mt`ahQ*qBQZ4YxTVGi}llAZ!|C$)hf z1!$UJGMnYt0!az_k)b1(4AO@-kFqEfM}&$%Z;xyP+vh1`8z>io7Kx}us48somaut} zS|WN5(qvF19mhCAioh_ixj4PZp71o&WZ_Y0WF(|<0+#7m>J2A{7a6`#WN@ir43dh| z<53#w3qCq49gGIp%Q%(U3Quzw@=Q(%Y$bLfxOTDCwQSS&hci{{u03Li>ts@?O^n$} zuTrh~b*rYRN_5oemX+QvRdHO~D_51?+mlL=(Nv{kj0E+#Qx)GHue-jD2Z*-SCYLu_ z8CIs$2-1BB31;_i5DIXQ!Fmp1;e7v(yE6r1-&8h@ZS~`&D=R4mW=J?SQ8UnP&qeTWizA~}Q$tX-50&)X*Zy)#%UaR9^`2D^4L$4-UE;N6B zUAZbZHU3;QvpV@p&F_2gokNq+(x~vmos%2KzB>Blk*TGWS@>|oC|DSdDD#)|yPlrD z_3P;EnvZHmX0P4)&ihxbngg6@3~1l%UhXmgr1=*dT4&I2?|k}a_AWMq92`4`ay#lj z(C*$)9$c6ZL+?%3eq2(~)2#pa(!korm_Cy;v^aY2vO1kwJ>5IdzFd6x>#h|~^B)Dh z@7DkAIy|+nZ+Q0Gms<}?-&+$6g(GA4v!3K$T4lir;;rI0?8}{nIQ<7q)U5P zNpxf4qz^Gy(iR|A!XV+Wq(V5I#gs@QdvAPqfFqqHk*+l6cR+p_BpJKWAWLf#GsU8D zYD>7A9*11$4d>KAJJr47O6rlAg##>=5lCc8c9g;084&?bXQ5m?Eg*lU5Kq~r$8jp1 z#h)5GB8}OqnB;z#!1p$Bf0uHNg{EsV# zsxM~Sh83~mr%DsXn^vnX{>1bs$+tT!rCLcO28vLyci_smaLz?P z2k`Yi5NkU0_?b>CxxY1@_m&k;*zIk3TB#N&)$+WNPCI7i8I}G(Wh5BYqtQS`Rh6Zc mMmOX@m=nT8Kmavexk!_ky&8Io&K?p^p6h)y3?pNwUlD3<~Ahmu_ z4^lm-6cqK;gE#S@77r4O{S(?_j~;sVDuQpircuOUdHeRw`+dyE&YYXI7_cqM9uW3p zZ+$1OF#`)SlL3s6lRx3=2Lj(HhvRSp#vuVGAq}Ts60%T$3Y>xSZ~-pDEL?#(a3C5- zFHwjQ0)R~Wgl!OZ?@ND&{6lS0q{cSriEvwlJ0jer7{1LGYOcK1ce6PWeO~gspQ!mI|+G7(-4Xpnrfy%W0IN#8C3$0 z#ks-1Q@%-3a&>5G=#+zTXmwac8@P3-4%KvI9r$C3asDjEra?%=BZ&hC2O=*i-zBL& zsTRP%MuJ8Eae5UQ*@SC=>PL}n~ ztwmq2=nmmi$sIz(R{kG^1_A-QwgSrZq8%=E{mw))Ox*OtcCgS&SQ#syu@Z*U4H{m; zE#~q?r&PD=^?adR_N^($af>CZP%L|HcB+25jqYid38;=xl9(VOV87cZi-NlgTC7??M2-uJ$5-kX^@a!SE~ zKQnk4;kB#pS2+>O)WKm6Aoz0fC!GC&z~?GJ5h`HA65N1gxCyu5HZ5nqNY%^Mxd35zzyWNhS zym0OMr-=JqzVY^Wqj#{e{%R6d9&uM5E}d%{vj@H(^03U|*Jz9tFZP#@O&=u__A&y` z6?QdHb0~)q2na-`a}b1(z%#uL{@X7S4tEiuXh9^K2yZ*KK(auiGFf*`)rux&0keMK z8Ou1s0QOLmHPo8whZrl06iPWnK@Nb3S-}q+Ww8wGAu_q+B3%^VG9ytVr38XQsUt6y zL90jEAu~RZ4&*+I32C|TG&mHV0YzUypQO!@Z4}roO|c@Q8B#hd zOF7Fd8dWKMenE0PiY%4Yk`qsNvSgd{b;~yC0Y#hgpIpUCIWH~B14v{DaZ34r5EWoK zu-Qf6r{`3AzHZhgO1AfsX;-aE*{g(0d&Hw-a literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cb037bcb4efabf66e7d228d53eee43670eaaf802 GIT binary patch literal 1043 zcmah}&ui2`6n>LrOTeWlI70$KmO#TUrfbu>KUx;li!4>K6c+^XkYs0TV7qOzyZ)f` zZ%~SO=)q$VJgOH3>CIIX#G{CM5vzz)D5#*~n~B?2iWtJj%$xUp^S$>%rjAW}6zCeE z0ljnk`;U}G4czNg2gu$#^b^jy1U7o`1fIcbcmwa?J$!^u@EN|qCM3_EGYY7L03h)) zy+-JVtB;vhpDFLSiObe(XP>l{!hkkwaZ8Ke= z3f1}^%K$3owD={FCGsr#p!fKpG@~i zG>uEG?sxnlC}@_2LNb(U`T=iZY(U1)pJi4Yu&8DZfo=*mO*4aS5VlMly|P`iB(R^P;L!(I{Ql`Of5NB0&UMiMsqy#i6nkv^|x;RP=t;w09LstDW8elI{)MD!z zEdu+cGQ-i?y1HMq-588Q9`JZ3-wrSES>Jd{7VXZJd2UbG?M~bcJGN^lN5fIwo)~fM zM#P)-wmorf(H0TUwOjtI$j6;)$9$n1#T*Zi&E39MbFMXGFS!Y2bc-Tp_iqv!h<4D% zwrFy`|L2`w0%LOCj~BYUGdvq-F7mh?&9yRK-Yew2Og897&3Yy*jTTBlx#2e&h2r=) n_r`)CES0@tX}lhej5Q9IL(dx-^>bzgzp&(m>k7aW{tkWvOij}0 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..690316c0bced6e4729a5e503715c5549ab0386c0 GIT binary patch literal 1197 zcma)6U1$_n6h3ozc6K9C8k&1MBh$E!&9IAWy0dY!o3Ka>t!&e@##B+X(ag@yuA`g3 z-HpX0VwRR*rKqiKS||zW4?g6jLb3iLF%g?qv3-!@LnPrg<-dzfpKLj|YNzk5!&mCZzd3vT*L~mK-oNIzFXPJW7y9a%(KC(1 zjbCj}ojku)b>&`KQw|Q%_&~e%q{(s|0G)cufj>j|(*1j>B`A0vJ`&1b9C%|;BVF3~ zmbFKBW{r3A(rNDI^7q~rsyE#ZI){6Y*8Kgouc>vcrTNumEywqLJo|e8kw5>EHcW&c z@_X(&`bXZp`oj*-kFU(O`8RA{8+E<_zKM-(&xyQ{pZny+$(g`jbZ=bynV-rVqa`OC z-`79y32%Q5F^oK3fmQtP^uq_GQH(HO6e$AW0DunjodOdP2k@!i|9ONM5q`0pZ2$t@ zxe^E`0yO|q8InYIB@MENx(^B-U?dTcNK8^8R@lY)BoR%1w>06PlO>T^mJ8ZNL0T*m zH$H|4>noEXgr}G3hs)0>viaB$7JZ)luvaCcRvJ%@6;8N=H2|cG*LWq}T z+Q<$$(|X)Y)0kk#C`MU|svu)t3W4CJD6Ir8B{A}dC&!TLjxMlTpi00y&vhY6q0y^A zAwkqP#~sLJ7bfJ0fSZ5~DV-Z)1GC5c;viE<9YWC>@KS+9&bO!y=`D%~X=i#!^^gwwjn_;)7JJu^3xOLDLx94eWIkHs5g-NE-aHl7mfhA zSwdo}n+yo$Onq^MWted@ld8%Zd(C!T{E6#RatFIirB2D_GCf94@hgSYIxC^n1^i0V zGSi8yQn#y5VG+%*bm;ADKVA8ijM?2|WlWk#>glZMFE&eecBGU;{z6S~zJ{IQQ>rKg zYf)YGe4TyJ&rsNU2*k!NJ=5J|=GL`mJO|87*6Qr=XjNLMO7nQ*Jyts5F~Y%6I37vr t$z-Uew${|D<8dP#(Q3lA2_sOQTpuwsEfCawQk34c!V4n^pqc)t9s`uDBd7oX literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..56882346e595e3067aaa68451a744b5903dab4f8 GIT binary patch literal 806 zcmZ`&&ui2`6n>MWHPEG{)EN>8vIKY8Vj4HwZubX!XhB2^V$m%Wv}iWT)DD*arhJ2t$y7B%FdQIUHu;C(g#af-OoM=%IyWIVk$`6R?!EQL zPd1)CfBAZI>&@Gp_UhVg9&fh>VM5AF?Ys(X4-Z$x3!iF8BnT#fjy zUQ8Q$)%OCop%pWr^`hs2K`FJw)&|7Nyp@$cU_Kd??!Q#TazG8GBnIi~q<^Mq_ zz~`{J6;P(?cDU5^qLEq{zv+bye{mshnO4@c;;Bm058Swu%VcwveATX2v*U$=XN^`W iPA+ea=L)Wq9<7e$9m`5*>|xcw!SUIB!B1!!|Bzp-n5bX? literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f9661fe1b8f3dc6051661f59daaa1111584c3f3b GIT binary patch literal 800 zcmZ{jT}TvB6vxlKJIV+m1>M_B46$5rS!>*8v~^WB(1)a?K%y)N!TGqW1M8}@yH+?# z^-%OGLeNt|4`mW4q(VfdB_ZK!PklT{g}(Y&Y@bGzCvm;}hp z2$@9W>%G~}RT?owLMAdm{Kbi1aAgaD8{v?HB86&PZK!X46eWptp^5}~3M>4eatx@@scux+w{={kz7p>z|(#r710|7b!x zb|wIHNg_w-&j>MqZd6bsCex|Np>W?yjpuNi`+bSL)6WMiWfG0D!i^!$T3No}7o7sr zS$V16b6KmQvy8{HZl1MX9c5w0)LG6N2>sY^^AeiIb`|6MP8^M@S`K#&wGQpsF^U62{RhEi(b}^I;nv&8Ad~>llrhlg*B48Vy#Rs*8gQ&+do#g}TJu_zjbYzTf}= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c9baf49eb9c946293095888191bed65921c0f8e1 GIT binary patch literal 1002 zcmah|O=#3W6n>MWC5L5E=o=CUvIH8oC0(1Y`^zGTpi)FyQ4mBS$BI@K z!GIW0lu(jTlW<j)QJWHD-90KVy8&(Hn>D9QMc|f6(M%lzh^Cks)YIq)h|g(r}EwP;nMy zg-lu@F_qMa`37?atWkzMn>SeTNQxwbY!~>hhG+O!6$`^88re+SNV2r}6Xt9y&LhQ2 zZZQG!C~q`bgXz-fG;)>gub}JI?X4kIYUbSV!bM215DEYK=?O3ez zqgc=Zsf9bt)n4i>*z0a8nK_Wi$^BbI2BIUf;(;ug?*Dan*1?!v_T!bl=uIugx%(pS zMoXQXSMW*&FP9JcQM;K7D>J1^P;L3GR;gU839lFgVWsMoE45}gU2L7JhMqS)<4>3k P`o7W^9xDKI^q2nu>F&sB literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4a5ea28ead0cff6cee3f80ddb56c71f42cda3ca8 GIT binary patch literal 1148 zcma)6T}TvB6h8B_(?XkNcrP{>efn6%|Gj1eWwOJ3Cub>#DQ6R%Bp3 z1!ZaY5J3e|LO%~-ND;lHB@|*k6b1QEQBO%k81)i%@2snW5DmlGbI-ZoIp25Au+1BG z>Il$d4h=Er$;|cdoQ%RGARsS5V0hghXi2l6Y+NSYJ5q8C9oTwNxpJ}9%8zuF`%7P# zH*YjCGr@eYxAEG9n!VMeT-f}nS>!XryQkhb&-Sk>15Py0>0|Te(ozP1G#FvgCkCAz zx_yRS)KTN$^wZ^XUu;xtf8Y0L&E4R1|KTf*MKNzR zodJe2;ZX@v(XYwh)4n)j5nogt1>gW6V;wEzLH2S%>`Xh&ZqNAjDm_%C`vZx*olg4ANKGh`h^CBGDin^zEWI|7 kFe6bt9El~(>e^IY)YSFr8lysrlW(o?!c`Wanf&kk1t0AZ0RR91 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..82ba5ea76d3794b3f1e8b936bca6642e3d813d41 GIT binary patch literal 772 zcmZ`%&1(}u6n`^W>ssO`onaRiSr!vg*u-p_Zz5Jtr63eh5Jc!^cT$&pwApL~DS>+R zpa(w=B2usyp;Qlgm*Tnqgr5gb;zQUFWzd$UD@{+DzBB`AMikBNz55RyJ$bhI{KYuhQ58D>mP{$e z@_hc|5iHYBQe9K0u~aT}_k}R3bI`eoJ1!z^U8B1@f8`CXvMs_T(%{>K?MYwfMZl`Y z!lB4*b_7#^DJjrs;9sS{lISYF8@VYcDTaYizo?LV6E z;_nFnzpJp*{9S_Ofj?HbnO6BXIwS9`9Sc0s6yZv4;N28mS9?g-t?2feuuDM}J$UfYU*HEgldVZX9F}L_edm4N=b4Y4y>Q9GfWJz( zkMP>hFQ1MWnDT^*Re;p?+23&WGlDa61AMpzE$Bc9cVH2Ea2M{ueRv3~@B-H16}*Nm zcmvz81JS^jC6EyU;N5;8yo2!bZy%SB{nNfL!3SC5oZSaQeAHiFdGhqx>e~9tjkg2N z2Svc7fGGhd1YiL~z^S8(bp9QhRE!%3uXM=s`f*9u)CMmQgNGwul1@M;ls{1sQglt) z9pJ-xgQ$3bh+rCI58-_g>!b#}s(NuEw4Ely3ScDxn&f*|B)~BqDxMwM3CJsk!Kqxs z&}kA3tmp|)xr*=M7@_2`Oa>^aR>!8sP6?z&R>xUxgKr~OhiW>q4rEioglx&U20|hh zNemqfMK&d}kCJ<+T7VQ^o`#e|O-Suv2Y|d+@M-c(##tbP2||-9*^p0)b!$_;%N!Oo zG9C9OYuVZg>Y3r7Igf%cHGKX z`HU4$IbFZ$#ob~qUvx@!yI#*1%4KHFIF4H^S%qTRbF(w`b0ycZvN=1gQ+{yb?7s6W IG|T^uKV)>MN&o-= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4d8d2c59eee2d2bf3da40564de2e5cd68abc29d3 GIT binary patch literal 1028 zcmah|O=#3m5T2K0C6%Qp>^u?(vIH8oknT3E`_m;J3JOv@6j3UQha{V=f$g@6KM zWqfz%`R4%xbF^1c1t>fz{enwb2HRWk2HwIByoV3)5kA2e_zvkaa6tl@F#sYp@D^@8 z__}-akj7mGlmaCIRRMJYrvyw2I4$6efGGjf0%izM!9ld}$IkAztcwyVeN#06nR43U z4Ur}C41D?Y=e>+5tsJy}jM8$Hq#u`bP2HlMcxc)0041G+&U5spNvx!6Z01l{reeA# z@UJ6-tGLRz&@}iO!ws=cv<$u?<5aS9fjH;RX1bo=3h@Z}9Rf{?8W$zNEG?CW*Y$>= ztQZCfC812q^TQU!jwCViXXvg41}gdxsJdWdmgo33$rt=y)N15|>R7qz*P33lS*}b>gwD9{2i2NW fsZKP4(edWVTHrXNW8Sdt(ifJ#a9^TdjsE}Nu!qfx literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9fcb8cee529f660beb1a0695fc96e23ebfe95a93 GIT binary patch literal 1176 zcma)6Z)h837=P})%QbefEVb{;T{&yH(OZgXnro9ZDN#0&Aa%0NDGbI&FUh65E$!0e z(lWPUP+?<=Xw@2!O}2p_vJWk*;)Eh?$2#dI^^^2Nl_FHOFQY=IVZ}f1U6!_NFg%Xu zectEq@A*B)_3nOM#ek1&r zpMRgu{5UpqcJy%LvGnX@Brw!EQv3GIuIce_Z~by~a`8%jpX&bOr-_d;6Fuj9F7ECB zOqu<6Q4uGswtHm()zH1%jL;(L%P#k=EjRn%a2VMlniaQ~?QdVi0&I zMQJ3li^Rx1j+{e^GrBp|=BNZb8?$a&DKxrw3JIdE4`&DRX$>aiR|nn$Ruk95CN?m; z%{)skA!QUrL*StV2~7KzGQ_Ns2&7bln@EYr9o0gdg|;$|UFj`SVw8eI`I?h#K5an) z9*bLB2hmG6%29k2q;*6|sLD5xA8N2fmUtWna`h<^Q=H^y2VJcw33Wz7kL%ffUoLS( zAJBva=A>+%$mnvnoHMiegeiOFimS~?%H4jioHF!uGADN*JSkfi)hiEb16Dk}dF8A= zlsB?Eb)>X(PWM)erH>Ex%cs4SoQ{ngYllzBq7aNNd7T@1_M-ns;-S+Z^klT`P+m9N z2C~h=dNyYqA8b~AYS5=Pd*XQ`oor5Ybp*TO;gpt21w)aDu6D-biLS63>WU;2{?1e& YoKRJNht?`a>9OV literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..50b6c3ed85e6464c819efd6129ded6d9fe2fca95 GIT binary patch literal 766 zcmZ`%TWb?R6h1RqWho&dbcS77$}*6U!Zv0Tb0tW{2c^DLQSgG|W-oOimv*x$NGJ&X z10sDYf(Uu=ArDe06n&`V7x?H85G_9V4;0U2ZL5gGa`v1v-*?V8XJ+-vEt>&*Mp?{u zzi#hk3}#}iG8G`dfAI(0jR|-(1M|S(0tm3743}XUuEI6A0XN|u+=oYytaG<1qzD0o zvID}N6Sn{H(@W)_lAj5TDZ-#H%}|<6X+{*ybo<55tJmWRhjl#fYHZJmlb zRmvfMBK0(jGH_zp9a@Wtbl^uR#`$;3RzS8$CoN_QW=*~kJ|(#cX;mSQ$J0pvk0QMO zdji0Jrq}{MNUp&X5X}c8Av3lmdQMm3F|tJDZw$l8#|>or)^V*)zqcl~tb|fJp~zAG zpM(NJNqe{j+RC~UZ4CW*sT-M_el!UCJ<~4Om4a>N+rzNqnQpCAskQ5Y69koNqv6}- kcH6Dh?P{&zxy5pDsqWf#vE(f37G9jZyCZx;tN8MN0nc`u;s5{u literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..50388b8a2ebb6cfce380f516f8023dddb1a5cdd1 GIT binary patch literal 8221 zcmcIpdyExV83$G?lopMm#kiW|Sha2L);lx%?!tlsEfMLK-8Lq$Ue29+?(QseXKrWa zvUj^Bn1<9xYB2?*NnL_Aqz};I);6_K(?}abTPY>czWl>Sqo$Bh7PbaN;&;wG&bf2% z%-+4M$%cfPGvDL)J7RK9Fhuz8J0ODZBKdMDB(9b?Bay4 zzyzIsH@N*%c$;nA>{;@BZ)l@UG|Q%XeH#kc)bqq~GsJZ5F?cRm>mPAk?3NsO<-4hy z)JJyCbCFsZoHcRL8MfRE^UeEUlVp|*13qCc4W^_V$>WmYO5OCiC`lG20WQSUNX9G` zvcwM6*Dp;Kq~&#kWSlD&*(!*nk;qWFp=q%o+e!eKmj*sqnq|5Gd-%QNA)B)_NRF?0 zFCAp%kvmKf)zppB%cIvp}S+jvp_2gldOu?GeC;i z9p>8o+=2Ko1J+v_W(w}yxDJi#?l4;0k61oYk9qjzbXq`#$<14K0SkUbB(DWmkaq@A zD7q2N)fBR9jsAm**oN3wCYhWTlLJn;%=^&#chl~5TmJqf-P(A3@Vn-Daa(vDCOXyLi7ee*vz2~ctbLFqLK8g z3=8TU0;tFoz}mecIoA}iHO-i7iqcT7F{GGq$U(0w#Ali;wqz5Va}Ji;31F#R4P={I zRRl@drgk+57M&Sry+U3ghFaJjy$ViDHdSmMhBZp;OcpSQAueb*SJ2axcA94$4M3-~ z#}{gi&a8AagdOu2F@F<5QrZM-O2y*bf<(KS+DJE3#6}S*#u^~0jkPo8Z#^K1*$!PK zQy?z(XZXm_;Tad+H)~AI9xJi|48B9bcPMLE*~GZ0VmES%Fb`&|eBL2${n;DbkT6*m zU_kZ9N0ncO{Zy=rvoObT+41d26-S$3NCk>83yZwbxYD3@ZqS-w^3|*Y3|YDZ_c6P) z(7M%BOM-1;8atK^XxTZ)jy?o)o`p;@N}nft*D4}!lRU90r^;oj$xG1DD2t^OGEBs^aM_L_ zhA$wRLq@<0Zm1>(H`11iy)l@Mp)mAf7h$;LeU<{Vs&7LwV&cH+(DMP|^$1 z&ALY5njG@C1n;+olQ`r+9h(f;Z3MJGSP}4T21ww`3|ct!wy4~HI2FqR{%OJ@MCB^0 z$b4M_T0;ShsaCH9>XK5qj|sK_{8Q2SYEW79goTjGjq=k1a-g=bxnB{4VM+6FGHCqe zK#Uy`N}seW;2QYqi)jeg1oi1yTj$z<>*HYxvp*KunZpF6KTIBRdBCl&Ok9@qbFd}g z{_yON><73BV&(`pRJJK$6QBPIba(?`cYuWVmaCv>g=?HT89w)+EA)ZMXoc z9~4Nrx$z;HdWTNQJjZLX>{)uIK*h&-2E}=$o>g(;&Lcz;t&7reZQ_+5kcSHP7N<2bp#8sC9JiFu4ON6K&c+I4W=q_- zW}c6MX1bCWIa=KY%$BPG`+~pbaJ*#X>f!nDQ?Gyfh=ZbQW7DBSFO;G8)6=H5L8w;8 zD%l$GhL*@LYp|VK;ybmFa`t(r&i-*=@4I~?>o@KkxmIa1ufF`@Kb&0t^(J}QN8aDO z@Zf>ldRKkt_z%{NzwxKfG_U^9t)HCOwR!j3kA7vey=!{+Li5258*ct>-{S1Vi+65r zcF#9;ez&}Dh}=9X1&4wnc2`R(b| z{T=q}Tkxj;zIEt769-zBl)iBGxxe2({iD8D_MCXG_{7eZcmDC`!*9L$%a_}qEpK1+ z&RqxZFYKGT`q1Q)M+Vp5eebU)mmT?*+w#njUravptEZp6cGX{pKPQz(4t%+3_3i~X zz1XyJ-V5c27QgWLPunKW>`%{*5>7Ke_ayukJX$=BX9(iN161 z-m&7?(ckxUO`Q2s&vj3~q84FJ5)e8$D|V)=!;&aO{W2_kCjj<b>20_0fMjM;E;|eR}HQC(1AP_k8-v9j~2OX$ncIWIRd%MT( zo?Qn*lh6bKif}+E6n=m}B5FyBApB`4B|iX(ZxF32R9Zk$DT1mX_`weZ;+>h@nYq2& z_4(pTR;29Cz8}x~G4J!r%2_qaeLU3Gyf zc8dj& zt3~7o7X1g+aR9MzPBOiuCI_5|nD@f^Z0z+?d-NTF1(6xNBAg3(Z5 zPGPqp%QW#)aY7$r z;dK28$HMMb-p*^d4vjQH;~HG2SaT1NW24B2KZ+0ohZdp}ZU>99yIW}}Z4(dh!ay|) zP#L9epc+n_C2@^2PJzncncp2$ET4;zNtWBN;i5wH20+-e(MG&6A3N1Z`c;_)bpZi1 zbP8bmUXh<`hT59u%r(R4D%TiNOgQ8q?+V$OrmHR4!`^~}Wex#Y=5QC;X0{YTUbdOT zU4lht23oI>SBRk|+9R*v++;J==HdF&Bq&z^a~R^1hI0}<%^0M4wx$c{jKS8Z(NM?8KjQQIMNNTo27b(|BgyR`L zHFR*}#C>zQX9Y8LHh{r*DESUmT)&bV7gOy8*;fFfl1~LgqUX&@Tiq zrrNp^=txTQJ|@`$$WK-0TR~;fQx-y|G%8LD$bt647Jfw_!cvmqWYEORfebrRls<1+ zAT`L<7tu_x#^~o@a?9W7Y?l1xA7s;b852W?YiOZ6H0k#CvADR8B{eUz< z&Aj9q=lh)eeSuUWlaW_QYE#gOs?5!S)G{QXOzjM${3bL-cql6`YA_P0LC!~AxW}XDsM*IL;OuW%cPDp`G z5p6^8qv$8N!?=91C+zH(tt`#!G7OFH5FV>$ekY~M;H0Qr zme3gSSZYUdW@tx%ZvZHwXLge&MDs zR;+q#?LCj(ebKI|e~;gH&c)xZ|L>Cj9q;{g^3}6G>Kk4%uw>woT|JHEk8c<@{ZB@B z{`S(nkG_e%^T_YN ze$Cqb*5)IBzUu02??3$6^Upc^pSga=)77D0^$slmTi^F@7oUFp$>VoDIQuMF+jHt! zy|3KA*gpS9zgYLX#pmuTd^L1nY~TH_|MBPtXRe+8%_Wz7_eHq=;NPEk_vMEQg*yu0 z+;Q*i->+Z()O&~DJoM?N#~-zO&RDwtz>T{r!~4H} zar4o)9=veTxeuJT=j_=ddoCEpp BrX2tP literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..0cce769e50f047c4764f93674602e98fe0811c84 GIT binary patch literal 8145 zcmcIpdyL$~87Co1iV_h7FUnfv$y8F_MYv% z>xG;INIWD!YLvFL=^qHBAkwBS;*UN+0t6)mnyLsSD%3%j-E=(a zBRj`kq*f+pJzNi~eq=J=_y=Q}*Kl3q{4L8z8=yO$4NtFb+5YHl}*Qk|A zAk=p*%}}IO9G95kd5i2*MAAr@RIV^B6XZZo0OM&8gJt<%1h9+mHIA(9TaX+#@s}a; zD=4ZGL^X||;ufm5fJ$5He&|Gw?;)GGBqCt}>07`*>^jkG@+{HH!c;O;>lq+*995)#QK^5%XSHpN)N~*dBeyHp>lblIck$Dvd*Ko)*F9@wo)4 z+Y)OK4GxMLb(a)4R2Y?UNNfagqCpgPqp(h_6pXq8BQI0a@*8S?umVc7?qqdA(}N!l zk;9eH5Tr!6mV&dNn2u+bXCZY2h`d4*1$+DmyDaZe2*;`TSBRk|+9R*v++;J<=HdD?BrwZ>ISg@0!#R(hrVr3O8|(r)eIUC~+jM4q zuq*7Czliyp3zFWiSktK2mlq`3&5UlknXWd9R57*z$>^?~F@HM&NzHcXB4&+5IG*8C zLkBlb+&9}iE10db0SvxF$#o5yump?vXxwbjfHY`ZGWlA54TdZyg!`CXT4?*tR7;L+ zY8r=r0B8j^ZqVbiieewwogF{H{CGT)JaJ|TcH z)z+0jM^c*iG07G{eyTd(3Mz}9vJf()QE^&84zw4x{3`+xmZA(NgC<@MWZ03S^m)qy zsX?y3mBF$_fytFzR4VAq2M|OVW*Z-LNV(taAM(x0cjE6O>OA!61^}Wkutn6Aav*N#iC?Y_@(y? zzf!z(i9YEAnLg=&UE)*O@i6;YfIaAyN*1jgl}1YQW!a|~qm`Z7V<;ulz4Erd^CY-7;s?-T;*DN%LJDk( zXd8kj&67Ep;&M`0`yjTp4iZsX$D8;f#^sYeVP~IgWocfQVQ74Z@K`nTJ1JELCq?D5 zgvN-+Qahq_Qk#0^C*+}ty=7@l4QT%_7AGy0b3+SZ%d)WvnWYjpsafV@qM5GbRgTuN z0ZZj-z`o#bqZlt4x%%k}_^CI(ebhlQ3a}T_p%==K|MWE0KMvIz`i;OMd}vAivIg6! zi({vN+OZ${F8JEO4ts~aZ{wNGKbl{C^6j6m{^zPw2cO^hQt^`GW52xd^785aUr%4U z=~ruR|KaA_H?RKw!zUMx?Ys5RZC`)k!lyo6<*qAy{Ks{dU3l#U#`+gd+_ho*+RZ1% zpWOeuQ~x;g=5I#5?|t&6C%*HyL*s8edG6uYZ@BKpJM0h6o#>f+=8?np?;m}n=jcrz z6}I$Fxv%f)8Qb*5C#t=B-+1*0wti@Lqvs1p-n#n1zrMd=&HT1|_xW3M`MkN$n+ z+E1p}@3*b_xo7`%#p(C9p4@QxzyEAM;B8u%I(~!ojmgKZs&9ShuKRc0bN%}(&MZ7M zb>-8qp{M>Z`25~irjERJ@ahlyUpY1Z@##-LJ2!vyxzz^_+ea48l7&@kuiJFG`j>^i X!(Vy%Pv(8=&9gsyYx|kCX$b!Zz!tjv literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro new file mode 100644 index 0000000000000000000000000000000000000000..c9facfde4c040fac3af8044e19b6889af039d5e6 GIT binary patch literal 8132 zcmcIp4Tv1o9rw;s^ei!cOHJW922r~wJ2RWh-O@BkZB8xid6>jh!!dn3`*v@<*_qAE z>|HM(+O(#EC=#V2O7LiDrM8Gf6K#Uu)Ng`{AP8Emii%J_3M~Xm`+x7vy!Uo?XKrtE zg(HxeH~){{|Kt6C|GlPi{PXKv7pr) z;#NbgynUBOpXde6!wq^K{#VcfvZLh$1YXpz8-1w355FI zrJ0Jfn&T2HJYkU~MI?=cMdb?9GC>ZE1TdZkF<7?mMF6|_UgOACeH)VF3jQ)geho#_ z1W`?8R9r{Z7LdNK>4#3__#Ubfmqa8iAY&c)hg~O{Po5=OS(tQFwVnae#L=|W9_J3k zhZ*o=-^NVA)@!dub51mk#y2CDPt;=>e!W;!P+@Xwe$c>*Un$As;0oer5QU$9=fi`~(8s#c|C&oDiyM5S@a&C??IJhdl5YBI3~ z(cqw{)pSXLLxoWVhop)iPBe(ZZWK0&or2L&VB}?L*?vpS4^}{l)(KV@G(Gs?5II~4 z4M9qDXDK*4iRE}!Wgb#TfXFK}QLx32u*>oeg>amTPmZh6SF_6Y-B!c1=y54EsaO#r z90ZuQ8YdzRC!XP`ZPhbFsEmxB1Yas>&SJME%QW#)aY7$u;dK4F zV`Fy*ve-t4G4lP6{oB@lnyIXNIZ4(dh@<25PP#ML3 zpc+e?C2@^2PJzncncp8&ET0RINw(Xn!bOGX4S=v$MH6@;A3N1Z`jy3kx{LrCIt8$M zugK3eLv2lq=9*#jm1_(sCLD5*cZKXs)76&jVQ<;NGDiU{bF7bSGdqePFWbzqKEa|h z1FcucE5uL}?U7e-ZnBwb^Kkt+5?B?$9EP~0;haEEGe&5ht?dIkVB(uME#{BICBsJTii&za3;dq8m4ISJ# zao=q6>|nmh1~B*zCEuZn>sNB)VyfLJY{EQf`L$X|qTaJNxglY)Y`}mTkB_Fj3^!A; z5w5@-CuL{1BU2r1k|7l+!V)avqj9@IBhsKv$>eMM4H&YV5bk4kX`!osNVVkHrlxV| z2Y^;kg4J<^;gD0)MWn20@Xyiil2@a81H-P(nuz z9EMIE<~$#HWR5=1_pYQOACMXeD5t7go#Z9yXq3fL3MmqC94^~=#P9_~tH@1w!6lu< z;8NOhF&~5J7z#r#ju3`BK4vK}tHw6uBc=|lF-?lI7`{stPCI5>V9e$@sm{Y_ZW>%? zGk7cx0j&*KO=RarseQI(Y>f@KoujjWuhSZrcy(Y;Y*0@E0CzZ(23Gh${#8@+fmy?P zx}3-8FmX1c`nF#s_6l&u#SvM>>DBNo2|^%=gQg~^$1tTHP|}0NRgO*InyklLf-g&A z5{Dee_eHaryFu9V8N(H_70D>sP?Bas~DYra6o(F+q2DZ>jBLU-a`EJ{{|U&b2YSBjS| z(I;ah(P{H(w=q!hBO+ zV2}?gm&-6;69(`eoJ;DY8hLM}`CEn_(H&hUq9>lKJK_W`)coSd5xO}oF>@3MvmgaRChQ@aYk5xOrlTu}HQdBNWXpDF) zwI@m^wW(KrLLS=KTb9<;fcF1lane#bH*^rTEF0U9St@apnq@vFn(0bj%`k9Qt;7#fG~FC!QS`9r*3)^_NUOc;PEE_4i)e zb^Mh1n|AHK@2S7t{N(KS?%chy{LsZ$-BFm@`cC@Pod?9I>5z2H~( z9DJ+%*&`cf{&?5KAI{o*-)#@w|Co1ZgzWt81*?W{A3Efo_V4;V+wMN{%+Y=O5B_Dt zFJ1|)Pgn0P{^%e=*#^3nh^Vd(^7CrpZ$Jf2_**_X*ANbjK%DXpx^7|`}Jw1>{ F?SG;LvHAc2 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..2f84bfe01b743ad7f90816345f78231c4dd7fcf5 GIT binary patch literal 8230 zcmcIpe~cVe9Y1P8C3sc^dO(jp$6qw=!OqOy^=_p|fu1zT(c06f zd)MRAXpr`bhGIx#nu3K0XR$>RsJR$uY6Zjy{z@?gi50*@!rnv z%w%G!i)~H#994WLpUU^U}ZvOS4QDU=P2SJY)-&2FY=~ z_tHUD5xGMIQB7HtUPaZWki4pFIl8M`CMpnvxWq{zWfk~`4c#3Jo&{Q2m}FJ7o&i$E z?hx1R=MKb&8Sqt0!%V@~zqS^Q>h2Kg-GEp=QIC1}<#bv=g~=^ib_ol9MI`rvE66*8 zC=^*lb2Wu5TciJ=GPWW1)k!9&#pHk!F7rOLJ|25HT_1hDkjq!JVWuaNs5B0Vd71{F z2et%AtqZI{G&snulns*dpu#AR9a2CLCmO`iZse4S7J*R_VD!o~r&$#-KUe_;TKiaC z(Dd*QJ4pAG&=5pK*Or2QJIU#0E4l0(^d&qBNCj3@IiYa?tAv@tG!zE!o88f`g@Y16XQL6WONL z6hTt9sXa}CMP~+DuaH-Wp%%7BuYwbkO%mM3VmB`8Hn1(*8m+4#q)46$ zmrj;&O~P?dLPrklIQk&Wc@{FsD1DynT}DLSCPiXXPF2X&l9!;PQ5H)nWSEG(aM_L_ zhA$vmKt{j|uBasjD`?Bb-WW{BP#Ai#i!j{rK1+dF)wdxTF>zq^X_BAC@Li;Esxj+_ zG26>YaUMpaL*P1_!F_QEXsyj^B0e`V)w3;QYizi!9i2n?YOOJdISA~D4eCJv;0|Zf zzzPekB(Lt#Pl=PzX zGF>BZO%D27g4s+si9-%Fu*rbkMnL<66#?I7fCRqGpoK$ki|YM{Q?)GMpC&9qRIajy z%r_(;6AEBVwRR=Ykd*3uOt1yupNh`cg36*NEQC~Ul%E!m1NDV1{E8q9OPYt1LE|q6 zV(f@e`lMw6*T7d_OhdRPXiP_MgKGn>kB2GD{#az^4ik|6FnPq~0k^(7aaqzYz?Ojf z!?QoKAK)g4nHOB+-aaRJU%-{{Wb`T|vMFdpRqW<~Yv~T5jO+}!{3Wteq* zP$1O%yH&jRq|8A;7f9L6i3W*fsg$kiN zdoSiBtD#>?XXsaim(J5Cr7PAa1+WWz%IhX(KMSx29bCzrl_SzfZay#j2xH`u8!8zk zzGPIPWSTc$BzU3uD!;%WA4Dz}W4;s`;Pv1{QpeTsdn?M{V)TUF(a;^bH?QVNaBcVx zpvCwbz2F2FSQXJa1dp4?b1=f?xUl*`tZN-4BDcP=?-5Pa8W2pjsWP zVr#@3S|Y!!!FHvLkD+y7{Wf_GLFLUb=cs`^R27dENCde*WDnr=Dv4bL);HzkOx>>vukR zSND6%7f-ZGt;fIX{O)f{_V#`7lhQGtIu^VnR)qVckC|ho!GWmeOHTCa+mBtldi4CZ z<@OazuW4D-vTb7WqVy$;Pb*KMu$e{s)|)YX%VKc2cknf}e8^?&`* zi4_lCcmH6|@i+f)>phzo6`;*U>n=$n1`-~Yhlk6oDAdG*-Nd)_I0=JH!kKBJGn zHM8q04{V+I?B;c+Rv3R-w|e=#d$<2^(;oNP>E~bDH8tbz7<>M}?7u#`XY2dlDjwN0 Z^WRqww4M4@;f>u>Guh|f`A{@8{{z{c>aYL+ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..3f7df28d60e9e941028adf7999c41026606465e0 GIT binary patch literal 8135 zcmcIpS&S4#7+%&FV-$D*JlJT_i3H=U&UEj>?kdr!U_?a}G%Av|rn_dh*`Ds9yJy!G zHyU+~2jLAQCivnN6z}G9oA$+!c5hF zeD(cR|M$-|<;`>EpB!2vq>3LsVXGER)QC2ug;7wqBCVkDZ+abD8!8pFibGsG)P^>0 z)#6V)T&3&be+4ZdTk1|gU}*)rAt@xrT;ChkCJTIzUeMw_HjZ%4E(pO<_8DTIi_64q zW8^T(@Mkev1U;;i@(u|jc5svC1d==y4coZDA zj$u=z=_PADTnk71$Yh}K4I4GD?$V$c7Mlv1O)9wVM%oblT(wkEEdij!vxw={t7Q@h z_w8FV6l)d7C1!ZsCi@hV^axFASA>=ca-b)GasI?$S-uwm>|(#hk+ppbSam7?GDLm_ zMI!`JPeZ79Un;gn-?7bd-5Obp(jKLK6oY{0O^@cPNDA)O@mBJ$;o`mhaZ9o=KNWt4YO* zBf>#|=~d%IqzVNgK2P6GqovXcbF5SvNU~|Do>4+=WZX%xQ$ceKyLDNnNtQ|yr2%%F zu0QNp*xkV2fhw+PlavOm9InPSiZ#y=IW~%X_@f9haA+Yq;V49ueY+J0(lPM^o*k$L z0V=)N2~>mWuq3WA;}ocL&iu}xVtg(DlPtGx!wZM#3xKd^qqTT#K6Yx5beG8vbru2C zOBBG?cSSzeboFYQmTS7+5!V<}%s6n6-wN4GE2&qqhrL+`%NPK#jKL1rW;7K+9@~t; z4#A>51Fcu!6=JA~>ycM*F4+wA@^Jle5}0Mc9J;^c!a0sRO*^yS%w+Tv!` zS9F9O%NMbHb3xMkm1yd<+U$ZvZ!@El+e}w4ic~SS0Lkd=I%D~^1Cq*ixJAq=iE!Nf zrt~ddxt-v2|=|M;&?joX{B|MX` z9GuXW1BanA41J!DJTgu{&(E$;MLr-E5>QQLo6X`SxzVWnrP@;vaWy=)6Nupph}y_a zbiuM_VQ>z;axrg%X&VYnFOCp~H$GM=(5uEVQ^| z<^$8u?sVCY(Pm-+llqo#6KgIcCsE22b6SovB$9pJd?xm zl;Gz+(TM{GnnobShS>lefxst;>`z7nats0%)oqRqq(6B~g8iAuPJc`w`UQE^;em|4k+_WXvoIx)`AGJs z<^wVWm3hfC&IN4#yg)jUWaJf+niRC-Dsyrmy$lE_Q!@i;zX45gJXDnzG-!!zRY1z$q7BB_$(cq^4X!_0K3_^iyJIFC28CQU5TFn|{EE7L;$&M*rDh?J3q0Y`WISxhTd zMP7Qp$Sb8wXPA?|JToU9uuF0(J04~~3$PD*rITqZN0pH>d|CD>#V9SeRWf>Z$rwt> zbRNDc@I?5Ae87MYs+P+LUlRfFJ2+R=NjLJ{O6#`_KjL92HT$a%o@z`oxl1_S4pZr8TG^n>Muc-#@|59->QrR~&3AU^l8<3f)aFd=T9}~}X zB(G|;wgZ@{Rs;0~e?6kTWa4Vq9QchmetpzNF$%C3(xw;6kpJ|wv40a(Yv|Vli}0o; z^~)Mer{=~ zzh&t~uP>RJy8qUv4~<>2%l1Be;V;8@tn85L;d}n<-uc${MK?TRFJ64jmcPFFYTn#K zpMH7di@!hk<&Tfg?7BnSv+urFcRu&i&7XdMZ13m$e}5yq@71%dQ{Jxkjp(=7@2|PC z|MYiv?A1r3XD)dD=t~E8JmI~(b#Pt(Ll3Swy!r2^&fWLqp2K+2Yd73<^ugW();s$b zEIMiFp$ivYJHNQ9^x};V-223lW#z-0cc1v$n!AhJ4*c5F^ILG*@S*^wjLO+tWSi?%8#X zi;>_F6ctqPfOvsoLJ|@YQR9V*F$e|`1pT2X1`!nTg2eMxRd-cQ&vftX>U&l1dvkT^)&)z>46HFy#k&@6)5575(T21j^y^lr+#LtD0K z(I+mh()IAaoaU3Qb=xPfw1S<0BRQ>1D!A^1+7SI*wNy|o0ieXSi0Rg=CE^SB z?OQVxYZcoeW-w=yJ&H+sgeJ8sLQ4hN*JHpqeDC-0{5LZRQBC=>?bY#OR(oKPDncM|MW(44?dU6yH_rIJKpfE}mf zjoKD=Ht~0$ifh_5r2#94t8k5C%`-%H8HFDFQG^&cv=HrJ93o1;-SPv;nD_zD4^)Ey zm7ebes=;Jf64!`v3RF60erHfIKIec*mQydo3y0_nfUsLeMO@6rP7RXoGTEWdBY=8= z0@(Vl$mW`^UQIJ{P1if(8bgX12M+REA)RRj^-6ZJJMUl_0|1sW*a6#&rXt8b2VZf<$jKqm$cAS1*c0F}487=&<~nkr4o>^efGvLBy^TlXi%f}(U6bf zMrt<1CFtX%?ey!&P+Ob0Nd<{8i->q@+!#=w3}`s+d@ZjEO_m+Nd(0j!v|%mvk`bFq zld*rU+rdB`P`^z-cOdQ{|nQXxLoRAsYSyu>#emA_Pb3L>t9$94)ad;!ria$;R@ zezP#RfL^(nx52axg{BvW2*Vp6sTAl{qZqOgQybPuCPgfUeTl+p#B39^*_@MVKa3{F zAa&M*N9GX7TA$TKIyZV6dt0W~SaaLlIt%!kqj88k3hIe9>hS@<8_ujj6dp_s)KIg5 z>1B7i?8j&`v5ZN5%PSLWF(l*QkgVkLs(XU?0f@vdLle@Yn@SHT>8|`r+amBxjz&|0 zpL;|n4jgD2f#e%z19SuepTx319udef2w0F)5L!6&wV=CTloIc17#7H_850ya327Gd zZ5ikh-kDPE&`RSVC!PYx0Mz80L1m|>Op**4R74BlKxIhl7#_BH}@=N6ssaH zy;tOw(517?N$*R|NeArWoJzKf+0O#(gKp_$#>!D;qzqq{eL^ux%Waj6o?bGBQZk)~ zuL?X7z9Anl;Df5=Qo`3n0Q?Tl6m`;#e7BPNEya(x9UVKMbMr=>1kXnF0bDWBL@!<; z4K|!;3xOxYlN?NFIcco*A-0STh$y4um-r09Ws5x#XOFzflDaM>(C7`}XVs|hq*Zx1 z2`QImG(tSG+Lok~-qa^Q77q>TEzN79LHoZ{oQzcV4NZbAEyf09W-HvJXUWIdGabpR z8m;XBW~b^yI(~|gQ z4W?6zqohFEu^;+Q+I_HIdhEL`uN*nKfA>X?-9OTGY~cqFy;*!OzwGWEFD^cGWUqJD z>X+^--2U@ckbW3=lH6L?N96+{ra*KT@M^Oef9b)k1jgzwjH~E|LglZ9$53! z(_fi4KYj7r#pmvOv+&lBE50ed`t10$`OC{+Y<}Zc_qX>K9{>9IkL!1=`sBdRmz}!% zqit?@bokpFKi?YM@Z0mR?CbvN_e;NgZS3$52NwT(X!rPuC(k{meYoLa^vwBZzp(Yp zdluc5e|*Q0PnSLVvHzL=&?9T#S+?t}-gExR-T6mxzuWiJ{Z}kM@V9ni>7N6qJZKCL zUcdT`@xO+b-eQxz`iixhvE;h(Ys#f1dlp@L+Ri5qZa1E~>5tn74xQn=yYKA|}){keRCg z`0D$s{_mev#XDx6caCq4lM3EHYnv8KREajI`GHq611+QRZ+fhfHkiw3WsBG)UmM)G zO$$G9aD}di|7A3f+*z|c0!z!-_DLo(X1mUqwmrl5=ou~CQ^EmG*##k3%05HvvvHBw zZEW1M{UW@@bC>WGUb#*r=MmHOXux4073^4L;s*ud1l|OEuA~_*@dF&xeE2HF$8sn* zY#qZUOVf+iI=JeOyFr10!Z&QxoSIF8W>{>>XeCm{H9OD->F27YoN5UGC5}l7POVZT zo^ap3HAAshwro=HPupZ(F-ecGK<$dq5<&L#2r$l{5G>Pm0)SoY*Eq5z*92DG#J}{B zTSmb+LDbVA$}XdBGbp#L>iSk-xeh83n*_wqAblC+hixmEh(3$Fata%2^bC+H4#uVT zkaTc9EPz{F6Eg=J)~-NxD;P(^>k#7;%~&3PE~^rN*)6+X1uJ>QEDu8}h_68uia|uV znnA8-(tl7DdvNxRB+F%0a=?m!WiO&n#y*#AjlNYX6l>;Y<|jr}dJd^P%|gzbwns>< zj-o;Ia8OXI+9boF!l;OSQbKS}^bked$gdJJ2BWT!ifYZa$JmrJWr z#fl@so`>mGV+EuF1tC6<-%Xoyxuu0Ixm8Cd)L+Qb{7$ z&yLe}$1D@u>-jrS!BuTLr2#945nQEM^9+$yLV*i^6d?u1uC60zcZ*9pEJND)2@}^g+ufOK-ejvQ9PQCof;(FRbYoY zjR5L73SjHIBAsixdNobTHC^wBYYZu795~2tg=D7X)GOJ+&a{JN^aEJNKnH9ynu;Ke zZN@-{V9}m|)+_J|G1SEM$SXLNY=(My*lwM8g(6@M-CuO!oW`A|_feiL?EpHxFS$@# z+|2sYj<93-B9?C|NP4dlO|4p;UXbW*W^{6!>FPxhE5;Te8J%5cEZ=rOQrQl-NTEUk z95(;4rh^A3-kTxE^d_pT0fT)g+J}m^TTDF{L%ogsD)fT|w_Nr~&^~*k7ZN(l3N)x; z`)J6=a2+)p;3D*K(suH7WT>r8)TDw$m_kInHEs;3PX;s;b-t!sfhNoH;XP)L7Fw~K zdP#{*rLpgNfRZ;UK2lxe)!Vpid7!pDx8Bl&kVxD`Ks!r# zCSf@^p)CjYeQON*JQq2nPCrl2F0Uf*kuve9rYaSh#Y=RfQTa=?ry$}mJhl^v;R}eC zkR9oQ#m&N?n_jt?x52axg{Bt=2*Vp6subu|!x+*LQybP$CWS19eX+u6#B4pZ*_@MV zKaA?*kUHzZLvsjZt;cF2nH%}W-j=B~*4#F?&H}#XXl&w)fqG(%dUOErhBIprg$q*y zHPm!qdfA;W`!U)~%wtmDbW6mX3(43xAPcy>>e)g(A4Fo8p$X~HL!}3lbXRtPWfFKM z$HFPWFL}|40|%N$AlZi503CtACz0%rMg(#U0v6;Hgcc5cE$A*7r9`_Lh6QqK$^=DD zLYl>VTL$vNJ5#D1T4@~QL{k76fSPn?5g)P9zz5g~TQW?YK&u97r$y0?OFTK-zCWQydRfC!3Yr1vG}qyu(QPDRVX>}LV?L63AYY2~OgQidL!b+i6{{?o5d zuKa%C$*sTq`RgAqPwe>Y-yd(7G4s?JxBk_mFB<%$K0N=&XV-l9&w=`!eM9ECuh#n) z=XZVo_Suh|d&ygej%n_OBU`V!%=_t?m9UWR4-Szohhi|<0>kIXdzI@33=8YF#t8QIz?{7PRL&lpJ*w%|DzwMk?Cjgy;bv!+ znb|w`$ngguqOFQeBeca>XbA>LgzJR_-3H$UFn z*`2w)?U`JfboagQkI(nV`+mOr?aZBvmM^u{JT54|{M0>yQ!ZkmORycQq&h-EaKCBO zvO-rXA>=fyXKkTt`#pm9iJ=t8dH7#Ku<%_a&BAarr|33L_`-D47#1cH?j129cz3dj zqr}_-6EyCg6x`F54A$$oI5^Rwj9TUz*M;BS7&vEP)wGDgo*@Or&~n&zWNbKA89wLS z^p4n$;*@Oo%EL!92src{#U@GI^UoSe(axKWOhIukY!{4@PK>5l3?_su&M76`5xU6d zyrdK_2>`{0ie;lz$Y3i_U%xbola|wTEZY|?GR=u3A(V;Sz_dt^Ezt+e%^wde)ifM{ zJ-9Ep$YxCyvZ}}Z(ne+uIeCnTrY@B1CTbHX)m=1g&CyH)WwDMOY$uS|4gM**=9K-< ze64N@OT6_IkfP$`nRYMfAU@Q9ep6Mbg70qWLt~ngN4*1x<`eOlg+G<#6M)LinN~sJ z{0d3#1y_)J1`#N_5Xsd9GA)(-gNlj;v9CMnsSe+pis~7`L`FnU zf}0$gql#W)W$Js$Ig#q1;ndAxO;z-*?mJLWioyg*16mHZDn){|>mkyz$T8uMAVk3- zg{avhU{Q2;OLl~9as#|DP;~-SVzLpaI>Tl$T)m9rpc388ZwxA$&k4vRRWD`Xg+t^8 zK#Gw?o0ZM+*zrb^TQUvm0s<(e2!OSFMSQM_d~2FB*F>?QT%$-);gExFSBTEE6yK5! z#aM8#qz(W}>TDp}q^cr_%QmUAL9ocoKkXHBVCTv(7uHM*Xb^BtF}ri^v7+DBkcN8aiC#g7;>lp<3l49l+oo^6#OHZf0WRBJthGF2X!0 zo4K5go%*xae<5MAEWm*3jgJz03)6!aOW;N8?I^+L=Kc z{mEA~3ovA9HoV95(L#OSBwAu@AlRopR{jG-{}Dh^WMjrUjz%&MLZ@rdyQ zt4EW(EQWib!l}e;D~#E0PV)0G8q0(0bO!gtA)vJut%>N|NLS9bl&#U>wt939;H$Pq z$Hp+QCpxJ60f0B0N&_oQ*c$Mr#sjmK_H;Inkzt~lQhn9TVzmjJ(G>@`y7a1T6k9fs z#Kn>jP>(i3J)oo)CtEcY!!tSTZ3!-g`R2%)&%wG=&f^Y!1S>&1=$~o?A&1j z(jO!bzdT^pS0*k^`UTh$Fn?I~hxP-^1U~cpXWS*Q@%sX%ge9X}A)!q{J*px%2TV&x z0A*-rz~on;8H9(h@`L~*kvo;7*3tivH7wnf(239c3Us&wV3&`C`z=#J&3|>4gM|6yb#gp}Tl5<|M0uU*g)puMjVtr%z&gq)#GX=lhh=42AwIKtE_> zO6IH_o<=hBS=omeBa>WL$%xS)a+j zhjH;_FR(MsT3MLaMHuS6A#SWH`5lwWf)k>0Q9?b$J*jn3I;M?(@_q79!QP^@h6c3% z7mH(-vbmv(utnKefy{h~8`I44(br5?@;pbY+kp9UHDF)x*BXqMlw9py1iuk`uMa;c zI+kMCWax!5bbor3_U1v-;i7c0V!l z%nhwyzvB33-keEodSyfNWf$Hkw>Dijdi2C&@`hD+Oh5isQs?J*6dr?Pu}{#Z+`mr>OIfxAM89;oOtlc-tIHIuRYLSIRE)07xuoE z|H=3@ll{3bmEN5k%&ffnb1&Zh+{#0^{oFUw`IAqbd;6UOQ#-etGiUqKx9D%I{`b#& zUf2O_{rQuDSG7%S(^0*!{yJ&n{Xrbji>=PslGlyL=`!n*Z7r zFP5Lbap|J_r%wKU*^f`369&F247_@D=%YmUzwW(x!~45$Yr%K5Yabmew`8W)FZ=k) z+{?$B_n*6JpQh%frpx)3C1=O8Us-keroO2)e_J*A=PxW9yDIm+Kjt5Lf77he`sI(?%RxGOLsl+k;lw|@4|n-0t2$o&r>T?L!~ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro new file mode 100644 index 0000000000000000000000000000000000000000..97a79ed3f6d9c64cf7b4d06301d69181ff4a8f9c GIT binary patch literal 8298 zcmcIpe~c8>9ao_UcwA_KtACuA*J-&Nx|p4ryW?&fJScFHsEB~ImRvS(XW!n9H#@U8 zGkcfQUfQ0ewNwS6RvV4Ut@V$ZnApU&2qleC($LcHd-LPH zo!yz+Tdv84gx&YPKR(|d@B8`g{kcbOT-xMlL!^jbn)jIGj+cm(ksQ}9Yp#@%yl+

q2tvs}y(J5*o4G*ysR&<&Dxu3KcMAd-eKOXY^9#e!@r0bpMK_+V+4=>qKG zeaS;MZ)uQK8@w+aWEGG*L=e@KLFu(rZ3@Y2OO~U%x@Dp~F^Ef?6jIiLf7sC7@!+#S ztCzy6Xgvd@gxw*o-A_7*4>RCiOT$dTgWEQvG2IM|$>o)4R>%A!LPHP{U0n+Lqa>@F+1xm!P6(n`p@D)OmWvIRcgTU`RD5t; zjJ}vvnq`!WW|ki3QiF;OLxgP`(^jLqqzDBOeje>j!?L_4J1WaPfj1S^Gfb$An4SbT z1vE#oQRZbDcqup`_popp)}XFoV~6(+6mdzKplQI$;XYiVSbH8KJ&#-q{wP8W99oFF zGYl5Rceiv;)Fv;$a|2Z`K&7M`fvPuZ7RS}kH~}if%lyWmV)>kcOwx>U9$q*^UjT&7 zJlcY{Bx5HUNpEFYQ0EXpMWz7O?iI1K-9C?dsJ10=Pv zcEr91E*vquYU-bl41*e0g2W7&X~orhemMPP!tkPQUe;Vew6a29kVs^}EEaY46% zZQ0gXO${MM@>IBVvV><6j)M|9a$v{N2Vu^$kV(eq=gHo6ipblfKy1pX^4V(g5_B|5 ze<}A26R{s2+i}G31w`}62zWt9H8EI4TQ2s-U^<4v(2HG!;f?oM3e2j$4ata!1FKJy z{49ohk;19OYzK_lUQUYhFd7>I*Vzp2i$g$bZB`TUxzSlU+cLJshTH1VIfSp;8iSaF zz@FHk9s~g1a3&3`uwZK-nwktu7whSK9;3s=az^zvD^Ij$aK^wcY4zyU>QQ1lKoW0K zrI328rqlyU`lfWNt`T@92mLL<<<4*tha9L{fut*T19Sxfp9Hi&SP}3w2uR>t5L!6& zwV=0Pl@i>lSQhZB2^$o?38^OYbqVMUwKJyLpp?o*POt^w28hmAgUX^OY?4%Nl%E!m z1GReN=`4Lxx?_D(0K34aoNi+FvjF>GHCHlY z<%l$ro6pNW!Wg;ax=KcgFBw%Rna0f*30`Qv${#Su2a(Igm@kC}cs)3g)NwWZ-HP(J z7(HQkG<1jV%`15lJRAN8&|>_JUT}g7tcYk0g2&C{IT+z`Tv+WO*0c^1kz40A@fnOu zCVQcsoxGJrd0mX5{u|=Os*>MvsXRClDi4;Ab!PHSX9`+u=G zZYiG|st8-0jTOkumbh`vJRbwibR{oxw7LzLEms5f1%IvKc*)4s^Ebk8#Qy6e4vMag zO@|J>P=?-5PdmE?pjsWPY-_|DS|Y!!!FH^H3Mc=%e&`Q`(6H}&Sf+_nC| z;-4)zbFT0H6YE+Ry#I+UZ(KNX?y2;z4?pyimFb>ar$^)mrl0P<`PeN>c1dF^MsI8S z&Gfv-r{&j=_w8Twi|?NP<<$NW`&Vmocl>YX25kSBZai&NU3kBt+13(C{IBMY@tCt6=99@UPX z%E(9CULJkpqYv8j-H*3Sz3W^#U@dv=z4EDbts8$!-Z|F)gUb&t(T{I)ckR3V<*r-q zc=Ytajw9bbwC#&uecr_U@{ z`egn}$36dTzu7syVRGO6$usA6cMdFV+IZL1Go`EVpZ(SkZ+&sgm+m>(vuCT^K7aA_ zlh;1@RQblTuYBff&#(A;^R9nOf8H?l-1)mZo?G{YiMA>G-i~LlHD}zFCl4)s^W@c6 zw{|bLH{G>6^Kt8)&g?HZ~+Lf8_hWyRH1U_Rk&J`=5vUqEY%Eh~w|w literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro new file mode 100644 index 0000000000000000000000000000000000000000..e96733a33279ffda09104a288feb1bd9755ad7ff GIT binary patch literal 8297 zcmcIpdyG_9880;jQLNS!vPBY3FA|r{Y%_E3>@vH}Vq0Wg1eV8cO$uGdGk4DHT=ver z<9)Ch>59ZZT5Gj2m{{vZs;s!NP2(m?Qd>zW)!GI$#TwCQLTc%QXiE$UE&u3u&V8IS zcjn%m9oA$+!pu3}d&O4l1B8RE%om^DEXflxI)jv|5B1gzT#>Yfukj?+awhT(@kSsno48lm>>R@;P2CI58n|kgO()Ml@h3|FT{Gc3kZQjwIft2^LUvSk4rYDl9 zG!BV*ng*Z8rUImP1=b)M9OSrFouoXdFe+f16cNOU1~If7*;S%OV3dXA^2#)?ny#21 ztbhWo!>lf7dU%Izq?bx?3^w^f;FqRBRX`Y+0DL8qFaUD2VX$Xm6UxWH#mxWHPzHo4V?mAXG+7PlB5Q zng_7%@-hv)6r9N9SU7caTvM?=>b(OMT$QG18nANMfvXg2&qJgYkz>LiMTmhz3sJKt zz@qr>md-_O@&dd#Q1t;+a=IO;`l4oWT>Xp_ppw1JZx1S#&nd_xRd6#g@nnn0t2c)J}Ue%9HC+zT!1-_%Z_hHia6Q?Ln=^&1z6;b#`OmEa)Y)6ldo!4 zV93&Jc#qkmg|>c-$04=Ksx!jDv1al!92u9#6OsjAfbR??Ciru)RS-`d| zbF!(1kRo|196DLTGYQ8*2^~4GZENE&=b6YLll1ds@3JED7AX;na;jp!k-P*QjnZGr zJ;OvCgvYjq7`}jL5$ORh=xihg8)(bL-WW{BP#Ai#gD|}DK1+dF)wdxTF>zq^X_BAC za4%9g^_Y#qnC<1HI1i)AGPurWa96`foN(nFg>iN^LdO86Sp#|ubM@o-VDy@*dc2?dewb^ST>Nv8x<*} z9^I6BKuO=2UZbf5p2=~4OK>?GPU4UQ4J(jz-EM%cK;V;r_6I8hz6Jpad<#MghrSl{ z7OYZ&TXo9Gzp=MNK*{xEsO zsbRQ4&6{7E&Nxe z3H@E57YZa&j28-o?)tr$m#l_<$vvT85nj4LpXA7m)ugx$nhnk2qn|L`69s!%~$vX2KgXzxft`M&;YLoCz3j@hQC`;{uZMr z?2fu-)4h2;Pl9K|{{UKyztIa$aDjCZZ9?$4c{~RrT#gHC9>k{BK_YVNye2-6ami#a zv@^?FS(Mks80xWTuI*WJeB0Ps^ZuRNhCVX0 z^z6C0FPy%sW7*|5ax+g19erZ+Thre^p-n7ZcFU>VTi&X^JXR_H?Zr2LvT5_$=U@Ku zv2Q%|(6)Q`-~H6Lau4V3y!7UU3tzVW{JT&7Z1|ar?;g53RlWL8l3>iD z{nsD4vb_3_imV@5HM#HP`0FcQI zj%UfUkG*@_mv%k**KYObFYg=6?)l=n!>9i{eV_3s`~2bZi@iN(p5LE7IaBCaJ-h3! z$DbbA`}0e)Gb`Ph-4nOGv-g>YosplCJ-4qs{==ayyKA*m>;8Rt?*m_(s@dv6_oG7} zobJ2f)ZB`rWW)V$;7yNzx^pYqTO81`FARM5y+g`#zga$c&o^f;-?M7{J95`{GCI_8 z`0szw-@o$oiU(ilTxK5`boB47{NRq>OTQi5sb$X{m%G+KBEN#JeSYe;lgroL)OBR= zPIX=9e~A9tqu<_it~@&Z%FwosyDwgO>FtY$j$J!HG(G;{+^eqs+D#j>T|3C=_Kvf& iudn?WdB1DL=r4-M6}`SdK^Y;6w=7Y8DeRHf?h4HVCuDz_NukDCbXsh_vYiho!!~n zTdv84gx&Z4AHVnezxV(BcUg49*r}5ZF(D;|7sqYp%=VPT)o_NXr$v(saMm}aa*V49 z1-Q5@DKUeqS-+XHKT(AwUJw5ZaJsZHE$b32jSGq)1sr3FrZ#gO0c#H*;Ospy!4&fB zf)He8U)bHJ2vJEXU}IB9iO{NRbFCEKyw6Qemqbm+0o#Ej1yznqh8dAmQ*VdQnKbPs zhAEh71HQ8HkyQ*1Sx2x5;`E%gs*o}gni(OWSR2-JYFfcT6D*nnTuh1!X~pDf@aL?h z5Ninll~hrRsOe->(p~q3TMIMR;<6$|jGH!D#+bxM7{PYk(0oDGc?U3yKQ>sRrkVh| zyPva=jcFpVYLWG&fwVX>6B5FnYEW=Kb{jyU`6Oj0CMG6IO75BUj-Y_>a} zIbJOahuP=}ASuC2Q13SB;Cx5`^_nOU2df`mirQo|ftIX9giknQdiio+%yfIk^JUUQS15|vl7^tefVNqOd#xYRw7W0dPitsrAOcIrJ3|=^d zUjQViG1MqD=Esf=67PzTLmfo``49#$_g#^nYdm{34a+s2FN$jfDPkNrXx$3_ObfAB zvMQ*f4pz7dzzSCv!M1Q#5#(cAxVlKNaL<726?laRYVP%DRd8Oih1ttP(b^8>dMHi7IXJ;BhhP|TGxT{HQl&QhdH(FmSmbpnF6mfP z#Uk0_#ktY2{KeYSCE^lzY}*mR7Z8mh#nA=#WDA3P@s%rBZ7^;_q3IP&B)}VQs}$%} z?HKYS#x|_BOtM)F`#gn{iPvn@`t{V-}vK)nL zb#~#)jz*ExW>8P0QFjgi-f&_KqR?P!z=oP1m~wKb(|!y$6Vr**7qyrqPJv_;!IWlN z@~X5|(hU%aw}d%YdX!@60VRD)aHcFu@Ju$_Q-V*++)f-gkTn7cX3Pfg2n0THWWO^a zpkolQK&K$MaPVt^cflyd*_AOYpj-1yQ0OEiTg(?^pv?77lxmSyG6y-%6o3Z6CZ7!| zIXz~Q6sAGhv;Yp|Ha7Yr0{2*g^f(DL_He)_4v(bIH!PqY=+P4o?wFu3A4>`x8&H4r zm|XVzB0Kysf#`S1!wwH<^qIsZq#uPT0nLZ9-!mVeA+XGIo^eaS=ARc(CzOmLCByrdOqeN|;=^YJo*RCcK48EHR?GQ>&$$6u zcW_=&N8QkO%d6jf{J6KHA{%&ao~e`I*|0x=E5@GaIV-5aj1$cv@Mw6HgB~qMjpaVX zoY4UhX>`^lK1^`=#hx2y8NJH9y3Qw1`wg+qDpTK4tMqU@QqIq)jks;KAW28Pu}{7u z9x~LMpI1+V_Wx3GG*a3(WC=FE7&DL=sc@s7DIXorcqGqiw1NW|sa6B^1%ET$_L7LJ z?y>M2vHkk6jiRXws)3tcC`0R~r}gDcP_2fR)TN@l#>c%<*EEiLP>PJH$?Bi`Elz;l1CT)gSn z>}USHY2Jk6-~asMlgh>g*U#TSe%GsYL+4|$%XPb6JyZ3Q{-NW-OS?Aj{!Y0XzYs|8 z862D)I&kgQ3-$4M;=;vC{O%vD;2ZD$*@AD@8!fv(yHfvR_r6c|R;(F3`Ip}E^4lA> z4?dbc*YFSYt4n=nhL*oSXZ*P_iMm%NJ~igf+2iis-}&IgvHf3rc*ok74ON?#tN+rL zeY|vIk88Mp37UFO8xD3Qr$VD^S)WUR=xQ0$@v%Nj2mCuS+jrgZEDq_t>+u= z?;kw;(0>o_{_;21SHHI~(c676@Ih(l7bW7A)6f31`yrL zKdwv8SoELXR|cw2>{xgB>FY1w=pDGUb3mB&x^k+dc3#QF=z>n+%4>J+er(aS>E$KH zlXYm}`Fp;7@eS>4Yv@E**AL&>qFw#^lgAF8sv7KkKf1l8@7g!|j!b(iaq164$GRWc z@x)`3f4=S7)csP)+PD6^@W8&-g)g6Yd}6#Oq0KOAK0n$&cioqFao?KLH)ZIV#?sc7 z($?gx{yX|U+&NHtbkDLKwRaqUcf;W|`}RgpKi4aN)+OIsKj~5Pr}MU68+f{N+MSn_ sLmzy%ZN z?w*Ykxhg{W_#5R%0thf)K?H3IirOC$rF^_IvpX}l zce_4cT*-=*yP5anndhB(pZB(v_FlI1iohBpRs6`p16nv$BU(WVLceZ>T2ABNth38n zLCgh_D~ue_qEB30rR(8;In5{g>b6f{X$3n0$;HMT&mGmKb9|4H)1p0P9O8^! z5Q44jGsQj!mx$BG#y!*D#}mHy6Q07aH;UwZVtGCdI0~eSUAsbput?m{pMuYoG@~U! zh{JjSUq$%XE(M3JW7y=dvTwwIasEVLS)LmL>|(#h zku7@`uxbgp~9$y15!qCPV^8(+$g9KD*>aSkc*dT(emmlKUe|9 zUWZv-P&WmZpF@0z_V+v4fpnh#kf|6u@$7K3=Y#zRD`g zbLv&MNS8~iam9)w!oH8`Rbz*w3I!oPPu@-Ay53uy(DlAJn}+HcC)7sDodi1-G$*iA zmt`7fsU)HIvEy{SQQN}KPW}#5aZQ`1G+^a$JFZczd4|X?qtJstiVy>b7NQ-DLqzGf zTfQ$D6F=bjfvO*%GV+~3)t?MY;uucleKW*8lDjUmO10|)u7kj^w+y^>w*&O2CUAAn`{ zcfdBYsR**zX7+ao7VQ~my#lWgLrq+dyn-{yW~!Hm<4qF3SOUzU`-?A}OSsdF4U}iS z9YAMnNH5eDH?z^(5q2zJ#PZDq$yl#MQ?J$L7bJR{nVsBbhI&yXim?SqW@pzK%eNhn zRJOw{Qmm2?N6mkt>EOYM_h!(w{HYpiz+fMW_o0&Gl`_x8RBxl82K`{st5gCKw$I-9 zg@n$s3Jq%1KAQ3|+(FHTxCDKiw4Ht(nQChjH>n^I<`5BYjT-~nAOjkVJ73GILX%|& z@E)^A3vJmg-)yB-yJpHzrXHC4ITEMDRpjmlrDJp~bm z;IW-T3|~OBjGR~(tZfzs*V8K(^EQ~aq0sc=5Mg-ZBb5TZY7|2@Vrs)0$)t$IurEnjS?jZ!NasdRV{gmU8f$KwTW0}Zb2JWdM?pQYMm;_N zc*B`Bh{A)Zff{NyFzeZ!F8eXsOe|wk-}1`DS^~*9I3(R%UagxTegGnIfvE}Uv5ra) zDCq_HZrdX8OpZoVg3o$HCk`BF8iC{+W&?Bt0-waPKOPatF$h?YQxIA>^tGV7V3ZQ? zY8V#Ctr-&(ISFYN^KBXE5#E_n?a)f&ASa#z$N<#jn?Yr#r%aMe8B|0I;6Q6*^FJaG z$C8)F$)Jgb11WJNBz@MfKzfiz&pe1RL3=)i+8i55fAW|F`%{sf{g^=X3-YML0~viI zaT)37VM-wLk?c>*2V@8;^WtZm3)t*=fpj9t$SWif}Ir=@U>UW(LxJ1DfJ^ zs4CBC&=T>kMDJz)p<`G!DPbL7^cDEv4PYTgf`2PrP_)Dtr%i<~bLba2FuPKVN(b%b z0EVc-Y>|Tjl$)O%vZc33N|zjOrLyOknE@4_mKhZ1rDoQoiA5SF&?0(eTFBoyW?=%6 zQnE1N=q^2rS;eZz%UCb+O6bx#=45P0&B*}l;+#shi`mZt?1OdE$*h&5%19Z$Ec=9F zl$P5n86&-9Or>Nx4__5{B79RmV891e%cX>`i2(Q=oGI$08~JV}^;?P`aXUJ8KqBfA9S~7Q$1m|&g3A_rBF-Lpl_hmu zN}$mj!q2Kv-$|?Ta1v53&1i&pWVJ0xC%vgpek>ju)LWX@M1%JKQgJd;**7!^wzL=< zkeRD+lb$6XW6yLXuWGcm1DLB;1N8-e-J-o@;_Ad@@EdXT`lyX!=wmmaO)r!o|LJLD z{T`^+z^nTf;Y~~8mo=D9Es2r>X~%x(>pJ_8sITk0@7;Iw;Tvj4ez9O* z*U)=taqxpvzr1kq;!pQ4-T&WvcfUGtaiZ{>V=LY|JovqXpWJrw>Epe- zMt=Y0>0ADBDt}x4ug1}XGk0|@>i%eE-}2WNk3WCa+Rck^-TK+y6Zq4WU)=P;zYg4d z{O9NY`T4gCKfCgcbKYZbfBieXuw(kwuF0#un)&=MtB!2>_)q?+f#ue+r?20ZKeBeo z1H;Qc&`&PxUis3Q|Gdz>{IMUsvwGl*cb_`!J-O!Y3orj&A9&)^KVMyaa?L##rrzIl J_QoVu-vC$Bxe)*W literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..2dbdc71cc3cd474bd49c68c44ff4c06be986f429 GIT binary patch literal 8301 zcmcIpdyHIF8Sj?TLTNTjQ#XQ3J-JH2I7?>ko$Wr_wL;xZDza_53o5(o?3p`fW-fc@ z-r0Na>~>q15d5R98wnUNgvz5e#X_~IQDa-wV%nlu358{tF}wdbdg9slI>U})sbS7`%Sx+ zk$Ms_DXS4ZV@o~5+a>Q40~hFd_+L!2$V5rA2pr8~-6k<#m~I+5X(s00kzzW%-dWSj;;@!vxsV1)L_q$0yeZPv7Hn#9BUdr7u@uY z*ba6|HhksbqZt$&c8*~ar|$V@4P3PIrjufzxEF>cqoh-#85W~4DMPZjq&rd%{alok z5G4Vi#864fC>7Gg3e?vxO%bGJHJzmFD;Aj)MA8tZsNBG`NRTbr2h7bM4=mL*9DqHz zFS*EOOckrDBere#!#ZSXxf^inFh)bojAmfA-Na)!@A~7`=9w* z-4s?t>lq+L?Buz2FX zAFP0UtpltsXnMGZZKSzMXb3{0D@#G!MpBxQN>4-T1R%N<>MIyB9jvpwLpB_z;{D@d z^u?@FO}$hwQuH{N>Q`(KB5YZhwi?YL1t^H%^Kfq(Pb4~1lZix^?@d|tj1wv&q9?&k z0nJIQmw1`_UJ6bmx>z`MGpDIoA9CM;0xn83G!0le+=Yu2Yu7`hWsqaSA4Q0PLkm%} z$HAiL?iTL~+vEm#VW8>;sN{GfP<4mR;<$PlCqN~;nco;xET3bLNvdATzzc`y3xKeZ zK?8W8Hg=+s^j3-mbpZjC6BNMey`pxm$zp4oGuLFfp#O9%!Q^ZQ80dtuC{DyM{ zJxyLi^Q^N0=;Srgg<7RE%bg8j$NWXi-d@lO;Tpa2%A-kptVdmV-IZL!7ZA-L-RA|jRuY5H(3XqcF_?~_F!W*vVR+*`mIAY?XG3ko#DUeL zNnRGiy-?wlV>Se1wwsgUJdCFD;5wVZJ#h$Vt;K30IyaK#vn^w5Y`Co)odftPt`hBIkkg$Y{&(bU?&bg-Vz=P^1=tYTDOH8VtQ24{5akT#cIwNDbu29mf~ zkpk+`PN@f!^x}A%rV@B2bKaKVQZkstAqOf}An~%@09}E=CqC`>R|I?w0uuNZgcc5c zE$A&+rTDkXmIeH3jSULlgjACGx&$Nx?To25D5ZRn<8J}D0iyGjpt9%*nooz zi_jBvM_se&-n^VA!L#9g04>Jb==mqOz_N%|A$Z(8o`WGS$Awi7VpZ!P5xI446Q9Gl z+GH=VGs#<7nAb%Z>b)UutjhTvm&$_^qH<9}J;Xh!bx}I5O?>iw@=(U!qO^tvwEq{2 zCSr5qqzXI4C+6 zHf%cdLK(V0Jq>q^Lbcjv$x?|sw1j?HgY8ta=hQWnvoHGk?zQ*+{G%0HmJHo;X0!44 z7nc8E=__ya-g*4Cxp%kJ5p zuGvQinn#`14+gus{&4d4lbg;hE4DoMjj=PY&nC~Ee(u!|E*;x`?qcV)cKqjG z2X?GnG`3{padYtepH{#Bi)RLJ-rfGQvw!*U_ya5ZuWEhxdk;^J-q`Z{<&VFTpIzEC zp=XB+Z`w2G+lHS!^tGwp_fA}V{iV$5J*Bn!kJjfpj=iP5cIIU?anpet&pz?!V?%rU z_vO3$I`W_U^6w6O@vhST^nFh}ziM{d?R%z`@%+G^=l}NAzPtCn_s7c*y>-+2*c=%(F>^!-@>6)=cV{bg0dhsVM$;5|$efiV>zVPvS9adHsfhnkI(l^5e-9qaHLEA?{p7mXEkC`#DIBQ(0XO{l A-~a#s literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..4f57b9fff1fe963049330c0f05a33da882383112 GIT binary patch literal 8157 zcmcIpe~28_9q)yn2hk>%oC#u%@|Z}Cc5~gGxx3t*xk$~ACMKRXxmqP$Hg9L&-c4_I zW;-)`Kbl4(5|h%ZEwvC^4T5O}QArD>wfKh;MM|haYy%C7Pz8VgJp2(bwBPsM%zJNV zcjopsS2zNhdGr18`Tls{&v$Pw{HU$-GT)paW&FgF16nX!A=;4U2VT_-w2a2zthb8V zP&T8LEMgaZZD{WSjXud`GzXXIdH7#O^T>~@mPg=43EMu&M8a&>8P^VG_&2)vricTa z@)ek1(eK6%uEtZI+rusS+x5amkC?7U^@cW-v165pALNM>c(d?au{J#72RNwu@G5lE za;T5&9Cwjg8J%@-#h-A4JoAlzuvc@cwhf;!m&P($k(6-N4zwZqT$PklB>^tPF-hL3 zmJ7ra>YJBlDAG!nP4fOli|kcI(n#c~Twz)w$etbn#?v4K%XFOpU>DzO9ND64LUL^8 zFMZ^eP%uFd)ii{%!&Ges<%TP+Zv~d?pdztJK>Q5ShrvH=TfuDfEYixtlrvQ886XuL zOi1lv?m&E)0lQoiGX+1`y#q~K!35g22eEvj9?S6OvRMTcCb#5zWvuuWle`UFLHrD& zPz)iOs~O~aCjAFhum`cPO|o28O%6B_Fz24r&;iM z>|lh{Es-^d1_uSzicK;cDvS!)Cq)EtqCpgPBfmn-7>v3CBQMju=~mVJUZlDC|EVHQ$HfXFK}QZVWU*k*Z$d^k?UN5|FZt662bcD3x} z>2WDFs#p;s?0J~B8Y>`WCzITgo+t6cY|P$h$&vrsdR@>|kfn!7>H_EMu^RY%}VL zAT8UB!4|=yGXt$x$ScH96YY^#aB8v{YV)w&Y2xJzfH@3tQNy{2o~HNHJloI$bb5bs zp*HBu`i7RUWBwxMZz@Q7pJGk5QdwM(Xg4!j>1MjxC}PFf03@TecEX_bag$-cv9g4m~1=}s8#>G&(kzav%Fz=R1J_(x7-spye$+8Rs zYB)X`@-o~*#Rj+lbDWf&+>Q)&w26jPpa=`Fh>ym#2K7sWwnme$>6T&0vV6FY*`4($8ZILvu2a>z7&p6*?*io8ck#G{<5n6D=kxfmt=QAssPwU=3+fn8om2tZ-^E8-+2O z=cGChqv;87oz38(I0UrTV>OYS8@;u&En{nJxUCX$Yr-v41(2Vr&ewy=qNgl`3~5xD7LWstg)RPyK!hbL!^xltmjelQ z#3+5*vOsE(t1qTOtO=Uav8~Cqfz&6%B(gsd+4;i+q+cYDx;&89*CsAY`bF3hNPlGZ z$Myr#1U2)bYn<XoSA9P5$ z#qlAXdJCs=GRGUS>;-zJL&Ya~2E}=?p4D+;;f4We5#CJ==HUd5>4072Q^9gD`&ocJSTB{#TRAF?l;+E_k1<9mxv7%TlS{@> zN~U%5Re~qXH{=Bd`Ji&S1oJgv0Pn%6q)w`l_g0+0CFl{|(YAcLH?QSMaBYMSpv8n6 zz37A#SQF6(1W%eLb1=r`q_D<8Y-k-MqO^`T@p+6(Cws!qUfIgxye`4e@DAazs^xc5 zstity$|VU65f7y{Md_qA^~#UPLk)XN(i$7k{$DIkS}Ny;I>MG@V+}G3C2mr)%*RMG zUCFB)t!V=m%GH2JX4R!n{Hp7^$N%-y71ur6e%nCDnui|g>cHma!aL7zU%6%f zOE*05mvbMTzwp_w+h%4i{PoTQcbt3ww$(Eo6AwPN@|xohZhU3y(UGoIPrmf&jXR4c z`mY@O;h#o#ukD^Vb?>edXSY{Q-91XK{oa4RweInOTW_zN-ubk>=I}3Oe{<~m@$VmB z{j+B#*Q{`su3Wl#ZFV5Ti842TzG6SWsdVF@S)eRU`;07{og>JNM4)tGoQ%nLFn@ z_dDl&=i9{%O@qfyWLL@3EQJffFuR^wL#goyq`Z_2IY|ysg_OvKwclY@yRGAkW(eHr7p*S@X$Ev&}g8HTNL5x>JV4e@?=s0Q- zAE3N0nr?5S{7M9J zeuRt@=S3A~?L#lY{7Mj2U;wBRh{?Q~oUg+YmXkyk>fa|=um-CzUMfHe0yPAINv#*q zWdT-Y3jtFWfC5a(oN-$q6xbYvA_x6qG~57XlDCZ#U*Z%nPJvhmiZ)OXV->�#LiT zUQ~l6#$~|@^N63E@M@$OY7$@`HepdEtT#=9DzQN6srVI1M`@LBid;R30L!qiaK2%kgQRw5+}0+wvunO zLSW#ym3q%)%Quy1S&F6PYUA0UKs%O>0mj<&n1KOxCy@ zlLuo;S5RV95GW)&iK%5}nT?~@XxPI@>+thrag<%O$LwjFCTcps@}h!@_MQRlUtQa@sE2Ir28XiFaI8U&O6{5+aQ)=lz0;Y$y--$9YR}KWqifJ$d zFvvy)mGWvi*@ePmNkKs4FZ&zNL=ho1S)~ju#`sH2=>_^%K>rE^B^j{j+WKs21Z4)c zE{JF(av#9@SNQ`oEh}3JDKu?iBn^qB;%UJb^F_m97Ab~y707ZSLTA?d&{lPS5yt&t zsIuBVz`j6;-WxVk1Du zbp@6Ht0;UUl6eV;Fs40MwEa7$va+fYXt&6rq7E7nJ3;p&oD_iEXjFWGRpCT)9U!g? z9E_vb;dZ)8OPp?ZiPPaIeMHZ^2tlIv!nIK(xX z=OEgejS1uP%42x8kza{&+U7SbkzQWf=64eIB-I)vjE{^qB`>3e>P4X-9{N0tkTy{U z3X_V0El>s=n|x=qLn?%4|7|g6WF-RFe zZG`#qFiZrs1f!<6O1U8CXq_-9U5r7~nWkJIeL0vYg6B-av)iW(c-BTqwNoTGsT;Nt zdB?}L!s^%v1Syi~DDp-y9Y$IJhD?%C5KRQJz0AOmt@%>S7EYca#m4;P9Uk+etFvKf z2?8mvNl)GYCOzrm(?kZa&S=()@DgMN1K_pxd?uEV56p;d``7hpzyFPG7pIi<-E6wHTlCJ{_+Hz&3t(19 zTgBt<5wC8%P!2qvk*%W_T>!`KD7`(~-30|tt)BMzqd#@mUp(%(Qdia0x1i+giSPSr zPR`z3zwFuRJO1EiAKr33xO~oraN)wvcUyxyqaU_jy&e2ySLgDso&XGo1QpDuirjO=^VRZ@P?t>)VQO1$#?CG53b$odH&;(ymR=r z6%X&2?fxP9@+P0cojFq1wjH{g0w0a(K7XKcUF+BNGvDu;zI3po0}NaG!P#XUw`vY{ zU0EA<-uv4hYnxo&`fUKT~m{;>o$2dp(-^epFSWOIZ24-&v|U}!+S<9bkty#C8oTp$^Iz5b ygU*|G&PQU``kO9YJ+k{j$L*v0{u}pOS^L}WMSt$7=$q8sKIZkiTW^d?F30~qK+=l< literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro new file mode 100644 index 0000000000000000000000000000000000000000..619989a8a3283c08923351d067a48609d3202a8e GIT binary patch literal 5072 zcmbVPYitx%6o!WIhzPboiw(icg2a#(cend2pg{_Ofbv)hBn6hcvv<1#voq_=?A8Su zgb+}{lt(~OqNo8SJVJtsm;fTQJOolu3@9iMDIlSiD5*pd@67DnJF}0r%g>#?bG~!G zbIy0ZnH-qk`Kdw}7lq*x6GjoZ+_K0bC!)wwZV?e9h(Sw-inS~aMd=jD&j_(>l^U(^0&pS41kjHeII=nnA<=8BB;!t32c*rYW0NP_2y7%@ zYlUF@aVylGX)E7!qD7%5EmsrI8U@O+ObpnJO-~qTHz;8sYg}WobOPE9s}qnlsV z@)dc|rnJh!$2iV{WhXhcP!wST|BZ(|g0+r3PZdYeY4zwmZPG+V2O!H!cu1ANdfb2@ za0v4FfhLO6#TZtr5P*C_ldrHclz4bJ4v;)S9CEQ#(eWBOkb3PIU{ zjSC_cjXnmj`d041w%R$hB^Eqw5iAYKrQ)e27xTp;5r`E-xe9_zn9|wieXK@xUw{(+ z2$G?)4@lo+V{$Cdh>57WRq=a3e-8vWAbQa&yG=a38X3t^op`lJiZ#> zDjb&Z{JgQNiOUnKoT~RT%0w}lWKt=6VOl)7(k1#?IF}-l{IVht-eH~z~ zhuZBnm)GgBTD=~()8TNt+^+s=>ZKSHALwfHS_fe2BTz1lBT>S;J+Ew_zK}|_J(%Yj zz%mH0&FZA{d228kZfsa$jI#PQPo&n@YBD@Yf0FKvlFmoOtCAJ*MlHvo5SjWMoRBh7 z8j39)g?OMWI41qh>WL_Xoj?$+F}R^DAVx!z*&Pu8o0|YoU(|uHx%eas<|JTx&8b7m z0%{`6mxE)Xs4jw<*(-Gfxr5$`fLhB4XeQUx5lCMLTolD~q~MwTQx-g9qon&O8l2XR z_=ue26JKGpY!rePNw*X^Bj^?*BY=%gl2#B!1&P1R!jGx>(%cqF-bRZ}_{lju;YXEc z&CyZ>T3(%=oB?!tGR3!286Y~NcrVIJ2yzmUOQOdd_SN`U%DN>F47Q9Yx8dVdQ&BJJ zTxyauM@G?2TCKaK_RLZyhejFoWPnw@8AUbSnV7WpU^93$ii($2@}u2s$FhR5Jhm%} zQBc3oJL1bI7~M~13zjojb}wdfTQsgxFKolKq}%XZG*{XMDfWs6O(}<&nKpeEfw5Em zQ!G_B7?T9}9@|=Bo`&GbQaq`_Fb%QW>}7Vh!|8Q7+-{e{ZufR_Zv56UINChEX$Jc5 z<@{;edfw`K{@kHeeMa}~)^hRL6?-oIdSyw==8KmvKA1Y|;@|$|5AJQOIu&B#(oczH z%ih>fzqET{KnlaPCtqpY^6`c5me%)JwxzyucySElqdv@*F%3X+WJCgrp>HTeGN4_qe`Q-hoBPVv2 z?6}vmZqv0Uv5{dzpEO^1f5Go3x*UVdla3zur+u0cop6dW7!!|TiY^j;WDELl}_C(m|FWF~eQ zv^sC#LhH1?^Blv5mQ{skFa3b));Q0)WYNvjXX+ldKJ2tG|6!G*?#H1g$8_C3=7~-F zx?h^rqe1M_xYu_6=@oA=EDH`_M`n?68uPp?rqE9xMgmz)f~#hL()D8mkUT5kSwH{R$A(o449|C^U%;T$i~Tx zrX%t=qai(QO9Q+2)tx9noFgZyfoyDPMwa|en<|MjY^o4zrU4$`xfPT&oCA*}fP>DW z_GoEoAS%n1?)um|GO}WH-4tSvm>BASCJ1b8F9MnYQe}*6FWqWjMF?Dg7S?D!NxOj5 zBtd40YX)XnC&-^*YaA49lm^%YB}>D#`G$nBs%c>4ykBKO4EC`5rwz;uz&QjwY6pP4 zF%2>72Fk`X(otRJN}m8>qo<3=P*Kt-=2OVxdHX0yR8Il-6e#78;R6LFXakr|BYJbE zs4h!NRKUt>grAo1hRqUb65t*7U`Zv!o7O<}3Q$feN!x%0J7;2FLHrl1o}g91If&gm zG(nNWbttq`?4)WIgDN#hc>$!5RvCE51dim05E4D3*_d>?GayZ)4pmP!2`rNDwnBRL zxN)avn&n$hv}Km+a`o}-Qb3P2Vn7j_o-r_MP{u%DT(MXl0kekn2ndWS){9L(l?Qi9 zcS}Z5*Cir5)v1M+1Q zm>iCTd;5ErJE`{&OmbjBzr2z#pNF_-99cG5?eoH`y?xZa76n^LpLLLDHX5F7e7j1ZkD7e%Q@VHPL=$0y%` zmWV@y7zC3V!3+g}h<>KAIbr}x9|PdFsD-fY*dz`XWnf;-xj_m5^%0h6!!U8wUWVG( zDz$>#qIF`Ra+rZOGEJ>OCR#9294}IX=eJJ*cw(c}+bJHL*NxeTw&OEfAv!h=!HeWM zinb9vhtUu~;gjSRgo+@umjV3vny=1mvE&tAY{pO9;Tb=UI=hCJBk=Ni^t27&(bFit z%w>S-47FaImkiWxCYMZ)bz-Z@v6^km9tf_CDYx)#>Z)iz>s;=UTtmjueOm3ArP;hv zBZI~njWvLEtQkl3-I;l`*%G^RrQJyv~^j2&c2jlz6@4(sy z3$|iDv&G}mdJz}qCf&qx@mzTq)VM1iw5A+>W}4b81{31^*Ep&m7@q_rZrE87p4O1b zqLx(jOR;b?6pi*qBXX=S5>^zsuWwH0KgVYVzu9}G^ZkQMC*NLjVZk5s&mDVr$LI3$ zL;sE5{?^OaSDZIe@lU3veqZ*rdHvYfi)Zg1?^u$`st=_GPs-slKg?CGEZ(wk;gN}h zk;QMApIt1>oG|`=Vz_(%&@^0CS+$~P(YEZ-MU~Di@eXC;($Ue)gI_!tKQglOmg&^l z)7A%9r#D=EeSOESpVwWONySFHrq*9NpV|N2OSr3R^5ad1^&Oe9EmyxC`?2HN>dmJn zwm)z}drvceI{y6`^lS2~&2x@yOje#eaNy$UPha?O?&{|{_8xn!^Uihhnv?$?zW2+8 zmwy?tUs-zJu5If#CG2yzKA$@!y&1l?{m;rx-CK?1U9Vog|DE6d`e@GDXJh*On-A1+ F@joZigB$<= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro new file mode 100644 index 0000000000000000000000000000000000000000..151ec4194d68cbbe4dfe21440da196da973bd96e GIT binary patch literal 4966 zcmbVPYitx%6fTbvq@-5CrlDwV*9HjLmYIFETfjirDoZM;i;;(Qy*qoiJ9Tzuotb4_ zu>Rtq5G80NCTb-m5y1p#{Gk{j0V8U7$ScI4!4L}x(MAm+il}&JX6N3S-JR_&N&j@_ z&iT&$&N<)tc15gzTtg#VCrj-LmqaPJ(Y3*)=F-U3<&qI6$#Iw4CBl^Ue+r6x0x4=V z!3zjKPVh*GE3U4!8;VY(p_;@e@_`gp=EXkze_Y~loF0jTDlduP>8HXV%d1K7Xawl! zBx)NUcXjz0X24w>TR>u;HM$^KVy~1`C;%-%pz!Mv;6)J2sYo%?%?D;Jfzwb{c`cu) zl15wae%LKNP zZ?r;u?6@JlXUgSUOtdUzi*mK`Y*3&bE5!h7ZF<7Mm_Z2xj&ZHUG6@(ntVw`lRBOG+ z`{MAzw*tDDm(J0w8&Ubx_C@icZkdffP&+ZNHPt#Oxgl6pAG< z<0t_$g6itha{vyYK0p_~Z zYJi}Yc1o2XSLmDwDB~rdr94w5kdX>J6vguu;Mv`i1D>@}iro|qPU}WoMAh+$ ztFRh23PFox8j7kBOoLGpfTfdU6hzZN;w~Ndu{B?j*CNTYwAh57s>2h0ba6KPEJdK@ zHR-7uz@(>Ce1^&Z(HYHoQC<>IP>5U-Jr=O5#>WblEqP!uWkk6xKTQo44U*2K7Rm5q z6y2uPrdJxvDwXnQl+n%-u(~s&sJ1&3i#8T)35P~e@v_SQWVf3!M^IkGW<|3J`tS2f zTp0zU`^j#=ss?j*Vm7Zu<7)N7Cd^2>jN_uY(k>`4S2SorIqb}oby)<)L-{W-R8BB9 z35eXavBEsH;>pq+sl_hM>R`RBFTi+%ey=YWV3{D}s|ok_w_iT8=-O}m#dr67_1v|| zJ8Ms!J-qx&rlIEf+53h*y7y>v(|rC?tG7*&Din5xz0_m*H1k;_3c$n4Ry^+ zW(OwC-MHwBn)@eAz<<+Xi$1$@piVi|eEReA;r~?cTQ^V4Ts>ve+-?2f`!zd{H~)fq zTOzCWZcSb6>=}+N*f)L7$%FgenOOVeU?czS$ebA$!f!U7ZJav%lY3duuG1Hex!Z3n z-u#36vDzExICpv64lsHX>>7L}^81F~DJ_$J%yxH#!pe~;yJ0R6m^?gsee~$5k${;yBn zIWX(H+w-I?E4FW*_4pmICnAAFNnp52^xMaKQJiw#OG^yTfr%BRpqh~apx6ZzbR0E{ z@tJaM6jeJoHZ}{nN_}*e*TtT|OEe4Q1cAuRg@ECJr$&OJmTn}lSr<4&3lbye;}%2E z>qf{pafXv{)-LoCloo=p2z@}}LAAh0(fL|*VZA&jLFM}h3;JLq)t4gR{Xhyppi!F( zs9`=@q6Y)>Fdt-LOy=es9fSyHg(2sKZY~`3K!N0Kpu`n91=LfZIsiEXD5$1IK>K`9 zzPVCVl_j%HhZSTHKdr(m=c`eZ0PU~_iz=bNX%v)Q2c@Lq7CAa326)L(5aY!nC*-O~ z0a{Y;&~Vb!^5%w=Lu%hY&cqUoVpGqU`lA?olF|$^p-$r zE)J#-`spw$nR4Zvj6&hirNE-`7u+5+QA9{JR!yoFWBhTN^aAB8pnL`VyZ~r)ZDlrP zf)WGk7eqJ|dIDhOtK@;rwbN?{o6xicku*e>ilY`^%oPp>X`~qPRUqvR5IUQ^52aW4 zDp=zVLW!350q&bbOcrrop(dnkRrDRuz5^bX_cRM>*0+zJ5zu-^v!fSX45^3$xuPT& zkE;aexT3&3phc0XL^96<4pz(0MS1@g6c!d00{IrLsHlZP#HK5L#LN4jHx(71r6t%y zTL*~iepaWGvgTyl?G$CRnVn|J`GS&p7J@|YJ6N)*48(kpma=9;K}4%P#oAljM_K9` zOv%Y$ybx{8%82n<-WSg{@+)Dly!lm2q?DI3_??J7iM2)%<0GI=$p~nn=Auv#4}B6w zNS-Ja#T<)*El>g+gM25nLW3yvwxl}0lxeY;?M|D6qMVK#TXuGiJ;&Z=Y2HK&Q^@~a zTvak~^}R7AsmD8>I+0(MUXb4Aud|njuCiY0d$-H`c};nF3px$zwfCTT!TGfZo6g;y z{qnCp3knvmD;mD2hi7@(vnxKko-%jafJschnO(=H?Htv;@Zk5QN7GAd9nbkYEbVyv z;+~vmHgEajjqx8o2%l}}kh%Lz*7eI}m$&WTcIeaXn+EmG=<#aN4%eyOhlWg=cuD#^ zZ_g%o-I{4PIv3Z>xRW>epS1N`m!*os{yhEddG>5C>DJVf-%Rb3S+cY&eJ%4_@sybr z=C_xHW?Hu|Ib1j5T7Ai}YnSPJkBWzXw}1b>h9i@!^S9^kJ)U`>YqzBbN><$M{?&`V zxsN*cznwC^%oP;cAKlsh%FX(xjtw7FH=$pq%h7Jwv|II)svh2WbZ+N`d3`qYzGHDr z$xgkJ`ATgU&x#J+D%0Mvr|h7gK6&ux`m*_d^!;h$x~q@f4YhlBJ-+z+`eO_GuN?c* zny;5Nu3A@mBBR^lp47mK3)0E^M>d@4)W6YIvT5uurHze^_ru=G2m6k`_vPpj8?0mI zyk*T_Iok81Wlfup>)Kr`UwF8@b9sIFxQ}ut&ObeC#tE+B#_LTx|MgWpKK#k^+4-Y4 z9H^@Z?i+WtDQk;!z@T)ndC17Ewz7v054es58yNcsV|w%4&GSR&biMn-yBlXMdi>_? ZU5?eM&Xo%?y4goH)mAk8*EYKJ{{za={7V1; literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro new file mode 100644 index 0000000000000000000000000000000000000000..929e6598ea9f2d611ee647b86a9ecfa9f9aad704 GIT binary patch literal 4908 zcmbVPYitx%6sApMi6*rr4UvS9%b><+OK0}g-5P?0)`-@MU5rMk>)n~VyQ8x+%gk)I zD=7gZXh=|uqJkzu6aCSk7{p+L!AFS31T+L<@DURM5flXsN_=3vGqZE=o!y=7EkJFK;=@Qq)sW5B(&O%jAA;V0h zWCh{JDH$n}!S$_NQ*j~(%?v(K3S^kNtfujANfU9LAx(s)tf}CEk!7GDn;CFt9N6e2 zYPT$LePN!T=&FpZAT`ZKS2QN}3eAKHuo47@`~(8B3X(+=8E(3zzycFE2X#}n@<|vu zBqlI2NnBP3jyn!TOYM`LYW_i8=CakEbCbHizvA_mf>ilT{Ed|UB0xq?E zfL|m*6LtY%k%Safk-5;zKp1Fs9;qTqsQGLX=`?Q-CGpZJ;G6=53{pLypa2a3B?(zK zw~OkqB!n5Ptc>}o3vWm(;3fg?VHXxxg1u=ORD^-DQ%M*q%o!QY^c2K~aE~z@@gXK88kY?dce*_wHKPuePSyy_lJB&F zG=1CwyJu?UTS>I86)JM|@a#}v9jnHGz&1T$VA`OB0pGZ6v0MVC4eJu%8|oV-F2Wl~V^`0I%zo+v7$E~}iQ#RPx3E4{#e3)pXgl%@lSuWiq!MUX!*yCCwp z+*JVEZ#51qOpH!+ad_IYSQ<)8B|1Sa7SCt15G#gt6$Hf$rL*RJcvjsXN5w=Inb6t? zq;LF~98^WUn6tMkeh;|sfuy1(YeEWa`=n$YuXj8!2C>9a;7$3U9PQ4pr;sY%)elBJ{3(f8^q#Jm(HwWjr`>5MlgU!LYvWW23OqYz$ zrUhiSv0urH*5-FCkzHQi<9AB-q|zFdjE{~trL5zH+J{3SGV~1?A#0)>6rmD@SRfx9 zk9_-DA_@^85JYMiGvoup`kCtHhyVz^1c2J2I>I($lPFl2fVnlN2FVB1Ls+~4!$eUd z1hu+Vst37F>qJ2LAOWprn(Bd!*I}Y4UZ@PuYoC1Z*hZoC;Zsz>=;^#K+EgW(=dQb zPqp|wl>wqNmi3~%q@ZFDxg>h5U|Wrkm2F${fN^C+xtVWMM@3_#bE!vi3>iiDXtisW zrt?bG3>sxLS_Rg&W)#(PXX4SOgRNrGC@Nl7rEhkx8}kK~RqR$QpJ0EW*JH~l7~M}^ z2i7o{zZLVCEgILV7j|Jz(lsm>&6RdRnY*Gv%gW(prl!pzFagScnWOT9@kl`Ys+|?) z=|((RmL+BVQXmit2crTXj)kIpFcgf$A~SaPe|aK4-g#yIvk(6C%Pk{4*X(FHeB`a+ zgLkyIUO4-vyzp}S2OsbG_L0v=!yETyQu~jtGOkSSUz7vwnU-1oe6*!!-HswtGiSai z9vK<_bmr#SC%N0+z57gl)joOq){9M%y>r{bZyY~8w)dW?$uG_>@03;>U8#&X*8e!Z>vyS4>qg6j_) zKe%D%(#z?|k6s>0ezJAdrO9nW#TO4P7~Xm8_D*i&`qLX0KQsK?y5jfu7V;C9&EMCa lAD^*u@x1h@zs|q5yLD8J{?^m_%5_K2_CCEi_If!*{{vBBm+Sxl literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro new file mode 100644 index 0000000000000000000000000000000000000000..070f738c8a08409fb27780970531ae9a96f0d6ed GIT binary patch literal 4800 zcmbVPU1%It6gK-3rL{jcl%j3Dj-^FNnwe}i*(@y@x~Wtm6*m?ngw5TVySo!-XErml z*{&tfmR5>=s7i&jN>GCMASm^(;EM!N`~ji*P@fe1p(4Hriho!6(;E$nx@Fb8Lf{1u?o|39)A^7aNEsh#dWCf^-w*YA$htbnAf?A#fSnu1@pGI%T3} z0huMPn=Z>b1An3;X;g8DhFl9(ZQX6mcSH!QTBb|9_v^c+w@6SxZ3@W; zHSFR+q#V?UL7FmG#svt6ELMoAlB`)N<%rGm4pEY+pF;jAR4o!S1d6KIL0HoWy}4IZ zpCu(BV3l;>rz^Z^s|rnmg2MqUR6@LI1JsxR<)xB!Ok8$~mK!Qa_+s@Fv?^{9yZ#;8 zpvdti6xu0HR<$Zom71iyASk3=L%}hDBY9K^NsjA|3!Ux_NZY7W^^@M7&agvGOAcFHu+Q@{3)H4 z%!*-1AUpM`#kP%W@HY&53ABzqZxly6At!^L4rwCQ0j!%23@Hs9#LZcjL9huAd{LaS zE1*_+gw2{SUqw2UFg%8VSe|GJ8 zg(4}WV8Hm>*&Iw17t(-L$=70r|5zZs$a@QUZ=t+pBMjH}W|I;W4J|&Wpsf0l>=v4?-i=583_mM?)e~Q$yCE{Yb4_MzsF*#+b zcCGAfRd^2t@1dMw<=R3jV*BKE8`e9_jwree(h~)KMVA)OEFd`UDX@jGUN=2oM z-*-e_b>)!Vsk@VAXH@q+HY`frhJBiXP=w8V4?c(vl#ik`qc8^)ffG{iNJqpWMi>NB z8o>)ifQWXcwKrk_$~XhyuBeN!y|^R}mSA8(&AC5{01XkA>A^2?)G>zI+9`E|+@*73 zpmLmnw(?BfKxVq|P#kZx0Wa*HBJjjUX?9aQIIkOX5k1Feu0k|y9D)}qFcdu_1O}re zfWjxqF9_8@<}M@n2{m7n*J8;lyx5GNp2IVKJaP8@EJxtw4e03^AfTsJe3{Dt(;4c# zI4^l@I7}{?9vk4Q;aJ16We)^X#*|z5H1$<H*f3je|X>XpZ$5~aBcXz#F?)aVrM^@{b8Z+jYocZ>AljfQ~Pf0 zGG-SBCeFogT+2*-c`AM3(y_x&J~wyl>eQ>#?|<~n=aUy_=4b!@aQNy6Zx%OSK5$c8 z+5f;%`>q2op8jj&$1ks~Y`s9<-m>TG-9M&Q=dXYB?|t)w8?D!`|MuU+gZHklT=_!2 TvU={1JJz<|b9uS1@f!aFKTrwl|NPy!H`)D{4G zxR+JfOu!8HLLSCs&g5AjWH?KOf(KRzYOoSYBySrf?#L-%oC4th6l|a%%*ufEdZBi6 zy{HCDOs)kh$RU0@gg0Icqb337VG|Zr!g|vvs9_c;J(UVsU_)|1RBQ#YUo3J$tBMj} z72^(VpwKxnC|ak;6&_JdRH;IZ7k~;WRRi;wg(KQ&2?^(LvVuBYACNYq7DrCD5!gz; z(F)%7;}+>X(^kH*L`z~gCRZEJ1_j!&cnmPsrpFAl8x%8;Fs`*&CIRh+H3>)<)mkq+ z`IDN)lU*{zk(dL|TWRM~kDBOXr(CZPP?e2Ut#!(U59^&A64K z$U{~@4-8S9BB@BNe1H|I4fzUaLy3lm=K;R0zOd!EV{Nnn;Joh zfvpQd4TbIlSbr;dV5VkzO(un=Er_HcvQ+YG@WtF}Fvuds(5?bmPk_+b=6z_Zx>v#K ziXc>2Z6DyiNyKEa;E}3B`c_5n0rNdj$%~b3LNeC&@o^Gb?`U?cqKhGQQ6N`TwBiYV zfR5`5ECN=RIX{wl5eP7>J(sopTUb<7Tm-aRbfBUR8WEeO`w@@mg`OU$_&lq?KIS?= zTxaCxI+;A$K|2|j(?#bnxp~fXJ@qVvi4HvIa5DW6^+8t2nhhlx?e^51C(VVFW$eLJ zS3k}J(c1J!oKM{VJlx2zs2*+g8=gq7FKshC5q}cvjUvuRLaUOK&_*plp&*|6B%F{o zQU(eWi-J8+0vwxuC-g)F!hs>M)>zz70uZaAiSLdu0F#FSh%f3u*kpVX0n5c;X3dF1 zN&spj%$og9(g<@Yhu*7n?#`ESbn+F2PntH(&UFE+h&CCmBw*^l4lTc^HO zyQ}}E9giI!$vwPm>Bc51d%>~VGvyPzUi|Xa&V_}?&YrxPz5Z|E8T!qIZ{BTrz21Lg zbL(?CgLUMm}6<6h~+I@F_&cmtxjivIT!ub#UrG9s}%TYd%=hkayOOBN78*~(uzA*B@>Xf>5a-*7_ewYId>rbD*oe?VEHgD+8J$vRiwf@m` z>A#x|_xe7Rx%{g|{l|^%Rl-v%FRa|RbOy|Ox~BKfD~Gr?bbCL`ud08xUT?YZYs;1) z6H?Y+?HkUn|C&o{>D}#xRM*bpgZazYw37ax94p$taLTc>Yv#e+`n=~{HGf{8aqrya zLn9hDG%q_b>7r-(q6IxiKj^6ImHrXEcrp7#dPe5~b6=TRKJ^5@;=sh)JtjW+&zwtV gyxZ&L_qy$Ob-UWK@Wi@VZ=6f1FDt8hJ*E-<2cc;QMF0Q* literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro new file mode 100644 index 0000000000000000000000000000000000000000..09ad8a80111b87a5f580454e112992741352fa25 GIT binary patch literal 5192 zcmbVPYj6`)6y_mR28NajouVDQrVOGq&L(Noqykf*MT-Rrt+fIYH_1)2bh8_FH!T6H zFuaQc<70+ez<0%QL>w(vs34$I)WXOcg@I8(9FS4)NfoVkceD5I=Fv9%%;xU-&i&3g z-}!c?dqLl{47yg}9}qo$7@${}mYAe)5Sj{10`%~L*OX)8=z#n?Ky!=_ijvF6un;}= zF_86&rh>UkVon5U$&XG%0*R8qaMkFym-nDJrM!oh7@h-<&MpCUjN}JX%YlN9qna77 zsUVM{>T_aavyiLSM`w9m>+guPQqE2qd!4O5eSLU3nU)Y35*n-uT>Y;!*dc;-jA@L4>m>pQv|#ZNPY-3Y6}20 z!AnbY4zNt{LKenkZpqg{h;U&Davtd7LIF1vNZtlY$|I+MdJ5F}A!h&ub+ia*uNTTU zSBk2#WXaQE1sKFnukcFwI@BaUJFLN?O6YGI1!dPkDXF+bjt+``UNRKKc(KR{xhj&M zmef0RgF@M2P~=V#T^>G^s8W+;UH~ej5C+;Y9Y<8IE+kyUh!W~_Wk9-(S{ymqO<+Cw zYAbkqj$5SkOn3Rl5-sp`F}WIeRwT!>{&NOmGqOA7)WMt`GW4&!fdr$fF!uPa8B*)&ZK~L^PywU@fkj z=UGT|=z%JV6+#H9l@HKdSd}loJd|j7SQg+sK`qUOqD9B4>p%jggf`yEq+veM6DZBa z!Sq5O9bzR@ft-_3C>**JSTz2E%Z(Y38 zF|d9?go43a09M{g9@tXhZ$07Xn7ytzDdMnG3OD&L1n9=_ki{uaI?I-TS$w(eSD07);pRVz35^{MHI*tCAoN9 zH9*G|1?B-QicB?AaK(>!crWw}Ma5@n31(>P z0C7FWX|-B%t(2XzI4EbH)k@_$GnLfM2ooI`Xw9Xv5cL6CGS@;uM7zD8ZIrf<%<3NO z=g49_5UtJXi1TU4#>0&aOUNUye$^8x^`#7kC*n_Hy-~#Z2xwI@0@|nrC=|p~pM(>V zM@mJp#G+silmN$|-w8btfv{o-tTj3}lmJ9;XyUsg3}DH}0K^ydB5X1~iGbx{Fsp|@-Vf)q)!6iFj!79%czg-nuK5LpGWzf8c7q4{Fm7EazmijDb6Iy~k_ zk!RJ>5(H9Sjh>_dG=B_L=9lR~LTnsh!xm zv7mj?@8i~3qhag(H(xEAmG(zj+B-MwZ5{6w&$*T{eg6xEnO|(^ zx_PW__qUb(KdG?)R%xpl_e<4;6Zfuf-8FKN(;L3D`>S!LtimehQwhDSRMh-GnPOgPLY#`56k9 ztZ066$X%ah-~M=JN%O+xS5{H@%sf!+5ALkpHNv~$NXuWR zQvNPpHhkT(0~PMKRROKAhi5r5zdl;{AOOHa=_F@pYA8Uo*(PzyF9qjZ1$vSCus%U9zyV^Q#Tp9)p7y zU%YC*;#|3Q@}_0mCw;$n?B0s|*EheubnSJ|fHCiH&06=edB~?-Uw*aqku$FCyPx^7 zYBqm&`jxIT=3iI8ej7z~$9Tr$!x#!X<+qU+;yLW89 ze7QL_KmX)*Zl0rJ>f7UweB5^Qz8P=ycO6@k(l~5C>!|q7F?-@$S?BvK8arj*m;oO& zEeJ8cUFh6)ruC!LpZ}RMcE%k8C*D)E;N6GWjys;eZSbK11E-bVRx$0)#_N@HTbhK< u8C~tC|Gux_V z$c=GZqo%$JSsA#{uJqQFmqANCC(6l&hZwqw_?v8W5powXbP=%svcZ|p{EC4ykN%bo zPF(6!44gRROEx&NiH8_CvhbU1aA0B=F>qk`|Lg7P^7Z%j?0qxowC>h3LZ`LYu9HVcu{jf6 z|GP=XhFo+Jw7qh{iBmS@f&=5<%jfNW-U&W$^KsARb5<|s1fR2bSabQT+0B^XvnCg9 zE}tGAaPM1ps?5$ zpdqYk95x8!5q63(XuvH+hyrZ{MkvrS1{+P7BN)V-!Hc||OdKfWJskaMAWiaOKb z(}kA11?WI6Oi+9b>T~b^i6%SbULK8Sh8`Uzaiek=Bn;1UE8Ms%W#E=)w z-%Z|p`4HC(yg)9QW50wYcy-D}6_lPc-wK>&AAIEYd|FoLd^mkm3C2gF1*g`j{PUON znhZShc(ol?F%V4!wb86O_AQgO>zO`)Vtv<3QTNc-ZH=%EE#59Fp^$<`8I`OLYVzd{ z6H(h%;Hnz1LUM1dOv!8#KzwP!nl3k;j literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..1f1d617f028d0c90fdbfcc904146a541b0f902d6 GIT binary patch literal 7802 zcmcIpZHN_B7&a};J{F6xFss8rrE$0J%NkD2he|{fMxB1bzPKJ!fXlIWzapc=zrG%fil_ zc|V@_^Evxq>4^&#Ugn#lq>A@nFsTF+HKOz@e&E&3Kq)BUn@+2&^cM?C#UgguSNex0 z75YRiC=RaD^YB+e@yO=7#M!M0Bdu`t_pMwF@HsYQ6qb8ihLzIP|r323oDs05>jVHh|6U1^ExRTJ`JV8lkK76hMhT*t`cZKORa-#nikjOJ(K=IHS9s`lS!r)<>Y`9 z0rQ?)pN+j*OrSqJ6+8G)ECf+ZQBbehq!4tFkQP|bqso3NR11PBa1x`(+*(;qzZY=pQl^OCRJT-jHzmG>`fx+HxVi$qm{sy z5}IS!u8Zo5y_B3#d!fb){o_H z0*kVnQn5E}QyAdcfvOLn(u(as)t5F);2LF|1eI2t0aPrX3y?{sT`xnAK(uiq?3B@3 zyfz;@*+}}8!Gb!A0BR})Fj3-s8s+DjCU=zc=9;Fp1t3F;35OgEyFzxRsd6WEurupm z>Ae7!-q%L9=?z7Ymu-4qn_$sVPwN%(3Nh4pdkiZ$H`#Q#dD!ka@r)8+4m(ubaAwfc zv>uvg%iDlX>&Y(ECY@PZ-WGPuU&Q>)1xf3ctf|*(vkMaKW_ml_Op_Z$su-Jqq_@}3 zn7^%nBxgHx5u-{1Ot*xFOj1JQ_`F%=nBGKMiGT|)f$W_rLcjJoX0ZH=Q z@unI$vtgU_U_duZ7?W@ul+cv}`@S^-d!CCNGESf8d)Fl+?~w}eC>1IjjhrLyXp}_K z?VbBM0K;|yF?<2hGP2`!ere-Azl^qA9InB14TYr_2MEKAkH`h=s*w%(h{+3U#BQQ2 zhVQc2Aj51utl42s%KI=H9|hOh4jzd^z(ze*6WO`ZmE3I^$zscG=E6Q#@{rp&saz*-Y-7V2{L2fdJFw;gP2i(Y|_ zaEa@Wmy+LYMLxv_|V6FU~&bY6bf4N0R&N&*u)173Tt+J$fwWTDOFT*Gp0OE z&orpoEYG08X6ji3ujOv&kQUK=X+mMA>4gr7l;MRAp_{oE=OvunFRh#VmExAu^hxW< z^hpEkVxLNugV~Kdd(bJAoVRjhDk#hsWuIbwLUKzbqh*(jE|pCC=F23Fo3D!@1oZ%}q-sK5!nl0G#_jA9tt`#Wgy4js(F_S=mE>qbst8Vsj%5i=OySmO znb0PW{FneF7+99p)ROgoF)?AO*c%#1SC)+lWTs2Lgl3VCv1WR$F7vFG4VW(90tN*? zOZbY&2-1Z^!yS<&z^T|j#^$^@rSDv`zx(@L8xO48cci{)?xA^yAKdrA=f!I}cCY*B z%cr;Abg=u|1&deSbKNh0zI;r%`h}Cvcfa@8oW+aYez!7j&BhLI{+0HD6IHyf5y1 zbM?+Me}6OoXXn(iKR&(T^pO5z#SKoramS9kHe5DwuJ-FY_WXsCcAoGy6+RZ@=`B oUDrOe=fuW~7Jq&3Wgi zR|=|P;~L!$|EY>g-e{OEfe%$|c|?tbS&qG`Y!|nm#T%|O97z1)MXt*whU3!np{yFV z%_{Ny60v=E3)UsA;TF%we#3)Lp}D3_ZDsocQ@7PikvPxBb#Kk_OU%aLzy-x_SQZ>% zrWIACLaMl7`N|Ysmn9WsN#Gx`4N|fjwK8$J`u3&ilC-L6k&?G>k)x8x3ife{%H^h| zg6wJ$U_m%TU=7Fi0d{^~36QNg1_XIJ_~;?0iu^T#sHQ2DKSkB5sBo(8c&2YUHmVSd z_{39@b_)E%mg#RrYmrtKrh+b8&j6`oe@$qIG&CW1AU@21dB?y^!TDEa(WdFIp_ygG z@`-vZ!e7YeB~+N)s^iwMCv3;7uO#qG$!mm?vFVv77m-6Cq>Ax35FkqQEMDfV0)^6JD$c&|y`2h~+FoQ51$ zmiLj?MV8Jqs=;OtnSm0DTtakn2Agk@l4+O9TM%*Xdr(-Bf)&Td7E2f8!FDP>+Aar8 zP6orV8a2B_w+pFJp>brf>tfm=OrO*skNJACrK}eU$W(D~U{7{4#IJACDZyhX3 zZ%X-zq)kD94+g49fJ)1E1Jz{GEP-p7aS~KoejiY=d{!Zo469Lr9)W1%M%b>PMZB1e zoopmMRboLsKmfG@1u$0P{5HzYHBIg)kIgkr>k2@I6cY|P7<7g7Oe@Hp)W-IKgQZUZ zSo&la*`_xYK~}culU;&EuXkJxv>< zc{bh!blO;Yp|1LYTC=$ik0wle=cEXv!odFpl3hXKlmXsiIvWkFhEszf z%k>)GCo%E-H5>TwgO{zkdyK}j5gQ6bvA>C;t;S=m(@giZj8ofTSl_jaN9gObNJ#Bq}Eu(UIo&_ z2K6WaaECK#V1)zRTQ)Ttn3Jrhi+PL=6VEV)Za5WU90q4B?32NO73DUF>jAgx(-lrV za+G>NN%!RkO@qKSxf+uDM>?Ox+j~fWr-dZ57mo?>6d~RcXyMS?BL4gk=lJ4it9OX- zur5?KaqzYTjB){tU^X-PwxnoWid;zIsO)?*s4RNYdyFoO3ey5|ptZ1rXZAcSc@a(q zO~~m}>_{;2ERHWUh?gg(fwTMebj-BL_JuwXCZ7GN$YzA;0GPh8Nn~(>E@0#A~@5I;2H-zqFvR9eSZdBBgktL+JMJ#bXI4_e(p;{Yr4l4t>(bQhm|@ zyU3@qX=C;x&lYk*$zv-=rh>wJQT7SeCnUF3GFp1c=u*jaZ@x^@xcR!cgdiVeK9*v> z!VL(XvondAP$TZ1BnL~;!@Hwpdi2RU&UN702p6Emgm*%;LkNsTv;|2M=7}6k@G&8* zbr4%x2Z<=Gi&HfrFJ@deVdHj=idL57W>E+8S4;~&XF6(n%ap1w|-2G#p{&c1|_tf0Lv~hD~&hU(D+6Nb>zkTO><=XVQ yM}32r`=*Q2FXaFHwOTHJ^4$+T%EzaV_4f~4Ir{qBy+?kVIQoA7XMKm7y!;OsL?f;M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro new file mode 100644 index 0000000000000000000000000000000000000000..a968682b674d42b94dc791d6e4aeb73973fcd00d GIT binary patch literal 4789 zcmbVPU1%It6t2;hRx$LUM$y{dPDLL~WM+TTWT9xxrnHb^+l?ZWu)RBTH#<5%>&$G{ zHRMH#4JmoDh!0ZwAZP_2>Vrk9G1XuMi$uYQR?s5qLkmhpMexCMXLjzrv%9lDecahQ z=R5a1=X~dzrwY&AwtWE4I`(6(S|TPsDa}but|TL*wTx{_3}mC zAn>@T6GL;Q%<;LV6J_j`;6y!;>p8kL1HUy}g*cP8iap)7(9_2bp{nkc(19HC(OJ|X zU6V3tS)Lzkj%^TYMvQLQLhLEq!v>-WB3FNjAl*WRnn&C)-Fjd}2wcXFr_+4$Zkecg zKxT>SmdCQr!Jp_*7FArLA*3$Bb4Oor%-STRZGN*fTAjP5!N(9Z|)a0 zU`a^}Sf&pAbcHu*SD{HzcsPWGN{BaYfEpH{{8aL;h0AWq_96v|UaWqCR>dn}FStV+ z6gk<1LOaFHt9B)>Qe%`C1ch{JC_E-`B&URs5NyGNKoqB( z3aC{PVXGF%SBVZK43A+TmM5CUMx{}7MuiUKV1DT6oum+Rx)Uh2T3~9Vh%1ICWoS-O zp%@WTFkt+hd;unk3u(xz6lgKSKN3nW^4~)KTc~I|2*b7g*`x%;1B(lyQZC;Du>V%? zz{>pe{GbHWW`Z=-m&(XIyI8Jbniv#=UWH(_#OZ8%A6ZoQ=SeMZ5)adT!1^YR$q7q! zYGr?`!h0xu4;2i%&=yh=+oz~Iu-;*IMA2oCz9{f3dbD`f41(jn0^10?u08`YZzGFT z>2sIv-?40VB8%uPx~OPE24Z*kenho3qV_}a4eXJDa2){G4=bs3Ix(6YP9&94IhB-C ziNt+=>O+7D2m0ilK)i{)p;_X%u-W^PyTWxe6l}phWv8wZSei3+-?O-vjW*boib@xM z;E4R{$`QL$cPGuxsP20lSd_X0`!oZg2%GsHd=MR|07Yp=VGbw;C!*f5j)+4f7z9%q z!3)KJh<2v6H(~(FJ_f*DQ5Ru*aY-C3&A`H%bAJ>A8X+v#gJ0sP!wj{xQ|bn}OXtKu zc^!!v$-aSr?}N8sfR>FF6Dq^DJUnacpv8S1<^ zFGXy)OfH!o8{n$pSi`bq4+K-jlw0^T4ODcLbuPC^fgj`O5v>lrQYWj_%AavYM_R!8 z&Wxi*?#wJ&C)gGajibV{s(-Ra%~&j`Zh={$Ho^Zs@5YsJFutFn2CQeWcqbO|T0AbT z7nrah={Amw=gPaF!Cdj64dsY3)7E7%m;~p)!BEA)L?j@0%fnDFWAAWT9_dhH5Z@H}BOz+*@ GnEU^9TW6L4 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro new file mode 100644 index 0000000000000000000000000000000000000000..ef9e9c5fc2f370ef040371b7984987eee8580041 GIT binary patch literal 4870 zcmbVPU5pb|6c$7ib%ToQh7h9H;VWlh<#w;Z0xig)6@3ft6%Tv?o zo%5ahopZkP^@-e$Id^s8QQKPWs0C8QyQQ6yJ60l6TC$01*_zZXnYc*57qO}5iQ{JT zxrRxgTYwYQK#ptc<}iHMEEVExvQ+HqmWiHzY9p%XZUH@-K^{7b z+M{byIw8vw-Sx2zVh)SZ4NHi<*>bUgXoASm#|hF+lpAx26Qo-WtO$Wi*miZAPu3|B zH4Dfraou!T)*1K_Z5%>nhiJ&PP{r2W+I(X|Sk*FJ;=NyGK@9eY|D%JfJaP*J1=OaI zJfLA0cOzv$BL=CaoPW(X8ju!FFs5qfj4 zs6I8F zXo4cg>riN?I9b&yM^$Q&@`9j{_81C|2^`72LP+w0?zqtD-hecXx}kcqNnnwDzZJCE z;|_T}(=6Y5qHU{EmurY;p8|TU5d(_Y^o)U7gE9spYjEh9QCMRHqi(HXehYVc3hHb?kYqINAw074&pS6R8eh-E?3`Y2YAk&aw=GO?cpo z;%2)HYL!RW9P{O?K!*~B$1o7f6OCe{Tq`>LLI-j%FLd}$Qh>Q_Hc)Ifz|=?{mkn1+ z)11OWkrGlcVEpZD4kn5VX~3%FYca!rGLT;6y@kBDP~Nf;hHHDXNePMu78gXhRJs9R z@2&QMm5FT=-4aY&5u~BIR8kY{VwrNWh(R&vRR~rKoX)2AkwtZXo{VLS#Km+Uu)c|6 za)YVbVjy?=B zZy}RZ=yQke-@&1w4MT|DqJ@gaWFWTF^CPOI5p_Nk-@q>E3f2K|y*w35rc%91EYTN_ z!$&eM_b>EP?*U9WaBEDK7X$T0?DmWj+kxFaH-2BRkb3+*I5)OfR|%}m;i~i5vxE&d z7?!e1SHJIxy!y%^!&CJq_1>uJd~8^ix(ypO4WS5|`Zk;p9Vs70sYhWRC;}&>-;th( zL&O*a(;C4IMSzHgrm;I>07{Ypa9`9y*misp2TL%ppyu2mMSz9~%e3K`IBFk5ZS0j= zL2l7IF;F?qKpVNHRvc7+-R?;mxfZG%O7 zv5?#1acRB4hWSZ1@mxGt-UT)GiU+MJN0^zWK8wM`IR7=4Dhehf0ht@NR>0Fzm@Mi^ zMZ=`@CK9p!cwa28^vk_*xi=QOY3;=NZ~y!4;>@DepC8|~ao;!d&fPq9MtS^%yzrLo z_oPdmUFwR@GL`%Ff)2Z>jdygrD$LFhB*McKo-~ZzF z(Tl4motFv^;gd7h3ug!aI`?kRt7{f%1IsT=96UU=skEf<{{BM`_r1O9w!|mP4rHbt z{OE%Z2iEQSb>VZ*?(UrW`TO(NF8zD@#K_?E)zYOuCTCWxee9KI-n_T`!^oZ=Cw)%Ac3NyL$cdmF;(|`{~N5J8SR$KkU$n8vp SparkSession: + return ( + SparkSession.builder.appName("generate_iceberg_dv_fixture") + .master("local[1]") + .config("spark.jars.packages", ICEBERG_PACKAGE) + .config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", str(warehouse)) + .config("spark.ui.enabled", "false") + .getOrCreate() + ) + + +def generate_simple_fixture(spark: SparkSession) -> None: + spark.sql( + f""" + CREATE TABLE default.{SIMPLE_TABLE_NAME} (id BIGINT) + USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO default.{SIMPLE_TABLE_NAME} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM default.{SIMPLE_TABLE_NAME} " + f"WHERE id IN ({', '.join(str(x) for x in SIMPLE_DELETED_IDS)})" + ) + + +def generate_complex_fixture(spark: SparkSession) -> None: + spark.sql( + f""" + CREATE TABLE default.{COMPLEX_TABLE_NAME} (id BIGINT, data STRING) + USING iceberg + PARTITIONED BY (bucket(5, id)) + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f"INSERT INTO default.{COMPLEX_TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range(10, 100)" + ) + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id < 20") + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id >= 90") + spark.sql( + f"INSERT INTO default.{COMPLEX_TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range(100, 200)" + ) + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id >= 150") + spark.sql(f"ALTER TABLE default.{COMPLEX_TABLE_NAME} ADD COLUMNS (label STRING)") + spark.sql( + f""" + INSERT INTO default.{COMPLEX_TABLE_NAME} + SELECT id, char(id + ascii('a')), 'new' + FROM range(200, 250) + """ + ) + spark.sql( + f"DELETE FROM default.{COMPLEX_TABLE_NAME} " + f"WHERE id IN ({', '.join(str(x) for x in COMPLEX_DELETED_IDS)})" + ) + spark.sql(f"UPDATE default.{COMPLEX_TABLE_NAME} SET label = 'updated' WHERE id = 25") + spark.sql(f"CALL system.rewrite_data_files(table => 'default.{COMPLEX_TABLE_NAME}')") + + +def cleanup_crc_files(table_dir: Path) -> None: + for crc_file in table_dir.rglob("*.crc"): + crc_file.unlink() + + +def generate_fixture(warehouse: Path, *, simple: bool, complex_table: bool) -> None: + spark = build_spark_session(warehouse) + try: + if simple: + generate_simple_fixture(spark) + if complex_table: + generate_complex_fixture(spark) + finally: + spark.stop() + + if simple: + simple_dir = warehouse / "default" / SIMPLE_TABLE_NAME + if not simple_dir.exists(): + raise RuntimeError(f"Expected table directory at {simple_dir}") + cleanup_crc_files(simple_dir) + + if complex_table: + complex_dir = warehouse / "default" / COMPLEX_TABLE_NAME + if not complex_dir.exists(): + raise RuntimeError(f"Expected table directory at {complex_dir}") + cleanup_crc_files(complex_dir) + + +def main() -> None: + args = sys.argv[1:] + complex_only = False + if args and args[0] == "--complex-only": + complex_only = True + args = args[1:] + + warehouse = Path(args[0]).resolve() if args else DEFAULT_WAREHOUSE.resolve() + if warehouse.exists(): + shutil.rmtree(warehouse) + warehouse.mkdir(parents=True) + + generate_fixture(warehouse, simple=not complex_only, complex_table=True) + + if complex_only: + print(f"Wrote complex Iceberg v3 deletion vector fixture to {warehouse / 'default' / COMPLEX_TABLE_NAME}") + else: + print(f"Wrote Iceberg v3 deletion vector fixtures to {warehouse / 'default'}") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py b/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py index dac6eb811559..7850560574e8 100644 --- a/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py +++ b/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py @@ -5,24 +5,11 @@ import json import struct -import subprocess -import sys import zlib from pathlib import Path - -try: - import lz4.frame - import xxhash -except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", "lz4", "xxhash", "-q"]) - import lz4.frame - import xxhash - -try: - import pyroaring -except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", "pyroaring", "-q"]) - import pyroaring +import lz4.frame +import xxhash +import pyroaring OUTPUT_DIR = Path(__file__).parent PUFFIN_MAGIC = b"PFA1" From 9b313330eb6089646d50337500c320362f5566f6 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 7 Aug 2026 13:24:09 +0200 Subject: [PATCH 2/6] Fix build --- .../ObjectStorages/IObjectStorage.h | 6 ++ ..._iceberg_position_delete_kind_presence.cpp | 1 - .../gtest_puffin_files_cache_metrics.cpp | 36 ++++---- .../tests/gtest_rendezvous_hashing.cpp | 83 +------------------ 4 files changed, 27 insertions(+), 99 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 41b16b95e50b..c08b2f29c2d5 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -110,6 +110,12 @@ struct ObjectMetadata std::string etag; ObjectAttributes tags; ObjectAttributes attributes; + + /// `etag` may be used as a content-cache key (filesystem cache, page cache, Parquet + /// metadata cache) only when it is present and a strong content identifier. A weak + /// etag (e.g. the second-precision HDFS token) must never key a cache, otherwise a + /// same-second, same-size rewrite could serve stale data. + bool isEtagUsableAsCacheKey() const { return !etag.empty(); } }; struct DataLakeObjectMetadata; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp index dde10215881b..cf772af18b18 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_position_delete_kind_presence.cpp @@ -23,7 +23,6 @@ ProcessedManifestFileEntryPtr makePositionDeleteEntry( /*row_number=*/0, ManifestEntryStatus::ADDED, /*written_sequence_number=*/std::nullopt, - /*written_file_sequence_number=*/std::nullopt, /*written_snapshot_id=*/std::nullopt, DB::Row{}, std::unordered_map{}, diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp index 56f96b9aa65d..e8c2dab275d1 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp @@ -40,8 +40,8 @@ TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) ASSERT_TRUE(key.has_value()); auto & counters = CurrentThread::getProfileEvents(); - const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; - const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); size_t load_calls = 0; const auto result = cache.getOrSetDeletionVector( @@ -58,8 +58,8 @@ TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) ASSERT_EQ(load_calls, 1u); ASSERT_NE(result, nullptr); EXPECT_TRUE(result->rb_contains(1)); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 1u); /// Entry was not inserted after clear; the next lookup must load again. const auto second = cache.getOrSetDeletionVector(*key, [&]() @@ -69,8 +69,8 @@ TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) }); ASSERT_EQ(load_calls, 2u); ASSERT_NE(second, nullptr); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 2u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 2u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 0u); } TEST(PuffinFilesCacheMetrics, WaiterOfClearDiscardedLoadCountsAsMiss) @@ -81,8 +81,8 @@ TEST(PuffinFilesCacheMetrics, WaiterOfClearDiscardedLoadCountsAsMiss) "Local:///test-prefix", "puffin.bin", "etag-waiter", 100, 200, "data/file-w.parquet", 1, 100); ASSERT_TRUE(key.has_value()); - const auto hits_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits]; - const auto misses_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses]; + const auto hits_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses].load(); std::promise load_started; std::promise allow_load_finish; @@ -147,8 +147,8 @@ TEST(PuffinFilesCacheMetrics, OrdinaryHitAndMissCounters) ASSERT_TRUE(key.has_value()); auto & counters = CurrentThread::getProfileEvents(); - const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; - const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); size_t load_calls = 0; auto load_fn = [&]() @@ -161,8 +161,8 @@ TEST(PuffinFilesCacheMetrics, OrdinaryHitAndMissCounters) ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); EXPECT_EQ(load_calls, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); } TEST(PuffinFilesCacheMetrics, HitRemainsHitWhenCacheClearedAfterLookup) @@ -183,17 +183,17 @@ TEST(PuffinFilesCacheMetrics, HitRemainsHitWhenCacheClearedAfterLookup) ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); - const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; - const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); EXPECT_EQ(load_calls, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 0u); /// Clearing after the hit must not rewrite the already-recorded hit as a miss. The old /// contains()-after-getOrSet path could race here with SYSTEM DROP PUFFIN FILES CACHE. cache.clear(); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 0u); } diff --git a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp index c0112b4746c9..547c1232ed41 100644 --- a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp +++ b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp @@ -34,11 +34,6 @@ namespace for (const auto & path : paths) objects.push_back(std::make_shared(path)); } - explicit TestIterator(const std::vector & paths) - { - for (const auto & path : paths) - objects.push_back(std::make_shared(path)); - } ~TestIterator() override = default; ObjectInfoPtr next(size_t) override { @@ -86,22 +81,6 @@ namespace return true; } - bool extractNForReplica( - StorageObjectStorageStableTaskDistributor & distributor, - std::vector> & read_source_indices, - size_t replica_id, - size_t n) - { - for (size_t i = 0; i < n; ++i) - { - ObjectInfoPtr object = distributor.getNextTask(replica_id); - if (!object) - return false; - read_source_indices.emplace_back(object->relative_path_with_metadata.read_source_index); - } - return true; - } - // Head of the list must contains all paths for files with numbers from file_nums bool checkHead(const std::vector & paths, std::vector file_nums) { @@ -273,65 +252,9 @@ TEST(RendezvousHashing, MultipleNodesReducedClusterOneByOne) ASSERT_TRUE(checkHead(paths1, {0, 2, 3, 4})); } -TEST(RendezvousHashing, DoesNotDeduplicateSamePathFromDifferentReadSources) -{ - const auto path = makePath(0); - std::vector paths; - paths.emplace_back(path, 0); - paths.emplace_back(path, 1); - - auto iterator = std::make_shared(paths); - std::vector replicas = {"replica0", "replica1", "replica2", "replica3"}; - StorageObjectStorageStableTaskDistributor distributor(iterator, std::move(replicas), false); - - std::vector> read_source_indices; - ASSERT_TRUE(extractNForReplica(distributor, read_source_indices, 0, 2)); - - std::set actual; - for (const auto & read_source_index : read_source_indices) - { - ASSERT_TRUE(read_source_index.has_value()); - actual.insert(*read_source_index); - } - - ASSERT_EQ(actual, (std::set{0, 1})); -} - -TEST(RelativePathWithMetadata, FileNameKeepsObjectKeyByDefault) -{ - RelativePathWithMetadata object_path("dir/part?.parquet#fragment"); - ASSERT_EQ(object_path.getFileName(), "part?.parquet#fragment"); - - RelativePathWithMetadata web_path("dir/part.tsv.gz?download=1#fragment"); - web_path.derive_file_name_from_url_path = true; - ASSERT_EQ(web_path.getFileName(), "part.tsv.gz"); -} - -TEST(ObjectInfo, IdentifierWithoutFileBucketInfoKeepsReadSourceIndex) -{ - ObjectInfo object_info(RelativePathWithMetadata("dir/file.parquet", 7)); - ASSERT_EQ(object_info.getIdentifier(/*include_file_bucket_info=*/ false), "7:dir/file.parquet"); -} - -TEST(ClusterFunctionReadTaskResponse, PreservesReadSourceIndex) -{ - ClusterFunctionReadTaskResponse response; - response.path = "/path/file"; - response.read_source_index = 42; - - String serialized; - WriteBufferFromString out(serialized); - response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION); - out.finalize(); - - ReadBufferFromString in(serialized); - ClusterFunctionReadTaskResponse deserialized; - deserialized.deserialize(in); - - auto object_info = deserialized.getObjectInfo(); - ASSERT_TRUE(object_info->relative_path_with_metadata.read_source_index.has_value()); - ASSERT_EQ(*object_info->relative_path_with_metadata.read_source_index, 42); -} +/// Tests below cover cluster-protocol fail-closed behavior for Iceberg deletion vectors / +/// equality / position deletes and file buckets. Master-only APIs (`read_source_index`, +/// `derive_file_name_from_url_path`, `getIdentifier(bool)`) are intentionally not covered here. TEST(ClusterFunctionReadTaskResponse, RejectsNonEmptyExcludedRowsOnOldProtocol) { From 088cf759b79ed066786b15c590505f4603d14e1b Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 7 Aug 2026 14:22:54 +0200 Subject: [PATCH 3/6] Fix Iceberg trivial count to sum required record_count. totalRows was aggregating optional column value_counts, which can disagree with row counts for nested fields; fail closed on negative or overflowing record_count instead. Co-authored-by: Cursor --- .../DataLakes/Iceberg/IcebergMetadata.cpp | 5 +- .../DataLakes/Iceberg/ManifestFile.cpp | 21 +++++ .../DataLakes/Iceberg/ManifestFile.h | 6 ++ .../Iceberg/ManifestFileIterator.cpp | 24 ++--- .../DataLakes/Iceberg/ManifestFileIterator.h | 4 +- .../tests/gtest_iceberg_count_shortcuts.cpp | 89 +++++++++++++++++++ 6 files changed, 127 insertions(+), 22 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index d81124e47572..66e90693acc0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -1089,8 +1089,9 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const || !manifest_file_ptr.getFilesWithoutDeleted(FileContentType::POSITION_DELETE).empty()) return {}; - /// nullopt means a corrupted manifest file with a negative `record_count`: fail - /// closed to a real scan instead of returning a wrong count. + /// nullopt means a negative / overflowing per-file `record_count`: fail closed to a + /// real scan instead of returning a wrong count. Do not use optional column + /// `value_counts` here — nested fields can report element counts larger than rows. auto manifest_rows = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); if (!manifest_rows.has_value()) return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp index b8b239408175..a1b8101de568 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp @@ -3,6 +3,7 @@ #if USE_AVRO #include +#include #include @@ -50,6 +51,26 @@ PositionDeleteKindPresence getPositionDeleteKindPresence( return presence; } +std::optional getRecordCountInAllFilesExcludingDeleted( + const std::vector & files) +{ + Int64 result = 0; + for (const auto & file : files) + { + const Int64 record_count = file->parsed_entry->record_count; + if (record_count < 0) + return std::nullopt; + + const UInt64 record_count_u = static_cast(record_count); + const UInt64 result_u = static_cast(result); + if (result_u > static_cast(std::numeric_limits::max()) - record_count_u) + return std::nullopt; + + result += record_count; + } + return result; +} + void requireDirectReferencedDataFileForPuffinDeletionVector( bool set_from_referenced_data_file_field, const std::optional & referenced_path, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index 41de844cef25..bc04f659440e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -177,6 +177,12 @@ struct PositionDeleteKindPresence PositionDeleteKindPresence getPositionDeleteKindPresence( const std::vector & position_delete_files); +/// Sum required per-file `record_count` over live manifest entries. +/// Returns nullopt if any entry has a negative `record_count` or the sum would overflow `Int64` +/// (fail closed — do not use optional column `value_counts`, which can disagree for nested fields). +std::optional getRecordCountInAllFilesExcludingDeleted( + const std::vector & files); + /// Puffin deletion vectors must identify the data file via the dedicated `referenced_data_file` /// manifest field (non-empty). Position-delete lower/upper bounds must not be used as a fallback. void requireDirectReferencedDataFileForPuffinDeletionVector( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index f3d8d378cdcb..459bc19d1524 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -161,24 +161,7 @@ bool ManifestFileIterator::ManifestFileEntriesHandle::areAllDataFilesSortedBySor std::optional ManifestFileIterator::ManifestFileEntriesHandle::getRowsCountInAllFilesExcludingDeleted(FileContentType content) const { - Int64 result = 0; - for (const auto & file : getFilesWithoutDeleted(content)) - { - /// Have at least one column with rows count - bool found = false; - for (const auto & [column, column_info] : file->parsed_entry->columns_infos) - { - if (column_info.rows_count.has_value()) - { - result += *column_info.rows_count; - found = true; - break; - } - } - if (!found) - return std::nullopt; - } - return result; + return getRecordCountInAllFilesExcludingDeleted(getFilesWithoutDeleted(content)); } std::optional ManifestFileIterator::ManifestFileEntriesHandle::getBytesCountInAllDataFilesExcludingDeleted() const @@ -610,6 +593,11 @@ bool ManifestFileIterator::areAllDataFilesSortedBySortOrderID(Int32 sort_order_i return true; } +std::optional ManifestFileIterator::getRowsCountInAllFilesExcludingDeleted(FileContentType content) const +{ + return getFilesWithoutDeletedHandle().getRowsCountInAllFilesExcludingDeleted(content); +} + std::optional ManifestFileIterator::getBytesCountInAllDataFilesExcludingDeleted() const { Int64 result = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index 7785e5178f73..334904ee552b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -92,8 +92,8 @@ class ManifestFileIterator : public boost::noncopyable bool hasPartitionKey() const; const DB::KeyDescription & getPartitionKeyDescription() const; - /// Fields with rows count in manifest files are optional - /// they can be absent. + /// Sums required per-file `record_count` for live entries of the given content type. + /// Returns nullopt on negative `record_count` or Int64 overflow (fail closed). std::optional getRowsCountInAllFilesExcludingDeleted(FileContentType content) const; std::optional getBytesCountInAllDataFilesExcludingDeleted() const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp index f799e50a11b8..4ab00a7daae6 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp @@ -5,10 +5,14 @@ #if USE_AVRO #include +#include #include #include +#include + using namespace DB; +using namespace DB::Iceberg; TEST(IcebergCountShortcuts, HasEqualityAndPositionDeleteHelpers) { @@ -59,4 +63,89 @@ TEST(IcebergCountShortcuts, SnapshotSummaryFieldsRemainLoggingHintsOnly) EXPECT_EQ(*snapshot.total_equality_delete_rows, 0u); } +namespace +{ + +ProcessedManifestFileEntryPtr makeDataEntryForRecordCount( + Int64 record_count, + std::unordered_map columns_infos = {}) +{ + auto parsed = std::make_shared( + FileContentType::DATA, + IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"), + /*row_number=*/0, + ManifestEntryStatus::ADDED, + /*written_sequence_number=*/std::nullopt, + /*written_snapshot_id=*/std::nullopt, + DB::Row{}, + std::move(columns_infos), + std::unordered_map>{}, + /*file_format=*/"PARQUET", + /*lower_reference_data_file_path=*/std::nullopt, + /*upper_reference_data_file_path=*/std::nullopt, + /*equality_ids=*/std::nullopt, + /*sort_order_id=*/std::nullopt, + record_count, + /*file_size_in_bytes=*/100); + + auto processed = std::make_shared(); + processed->parsed_entry = std::move(parsed); + processed->common_partition_specification = std::make_shared(); + processed->sequence_number = 0; + processed->resolved_schema_id = 0; + processed->manifest_file_path = "s3://bucket/metadata/manifest.avro"; + return processed; +} + +} + +TEST(IcebergRecordCountAggregate, SumsRecordCountIgnoringValueCounts) +{ + ColumnInfo nested_list_stats; + nested_list_stats.rows_count = 1000; /// nested element count, not row count + + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/10, {{/*column_id=*/2, nested_list_stats}}), + makeDataEntryForRecordCount(/*record_count=*/5), + }); + + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 15); +} + +TEST(IcebergRecordCountAggregate, SucceedsWithoutValueCounts) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/42), + }); + + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 42); +} + +TEST(IcebergRecordCountAggregate, EmptyManifestIsZero) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({}); + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 0); +} + +TEST(IcebergRecordCountAggregate, NegativeRecordCountFailsClosed) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/10), + makeDataEntryForRecordCount(/*record_count=*/-1), + }); + EXPECT_FALSE(total.has_value()); +} + +TEST(IcebergRecordCountAggregate, OverflowFailsClosed) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/std::numeric_limits::max()), + makeDataEntryForRecordCount(/*record_count=*/1), + }); + EXPECT_FALSE(total.has_value()); +} + #endif From 69ab13677eedb22ea6ea6c9ee0ae168daa9c9ab2 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 7 Aug 2026 14:24:05 +0200 Subject: [PATCH 4/6] Remove isEtagUsableAsCacheKey, this method useless in backport --- src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h | 6 ------ .../DataLakes/Iceberg/IcebergDeletionVector.cpp | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index c08b2f29c2d5..41b16b95e50b 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -110,12 +110,6 @@ struct ObjectMetadata std::string etag; ObjectAttributes tags; ObjectAttributes attributes; - - /// `etag` may be used as a content-cache key (filesystem cache, page cache, Parquet - /// metadata cache) only when it is present and a strong content identifier. A weak - /// etag (e.g. the second-precision HDFS token) must never key a cache, otherwise a - /// same-second, same-size rewrite could serve stale data. - bool isEtagUsableAsCacheKey() const { return !etag.empty(); } }; struct DataLakeObjectMetadata; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp index f7d815c05264..892945961a50 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -206,11 +206,11 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( if (!puffin_object.metadata) puffin_object.metadata = object_storage->getObjectMetadata(puffin_object.getPath(), /*with_tags=*/ false); - if (!puffin_object.metadata->isEtagUsableAsCacheKey()) + if (puffin_object.metadata->etag.empty()) { LOG_TRACE( log, - "Not using Puffin files cache for '{}', because etag is empty or not a strong content identifier", + "Not using Puffin files cache for '{}', because etag is empty", puffin_path); return loadDeletionVectorUncached( object_storage, From 0924819ebd7220d1f359b21efe8211712ad25ec9 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 7 Aug 2026 14:30:30 +0200 Subject: [PATCH 5/6] Fix settings history --- src/Core/SettingsChangesHistory.cpp | 88 +---------------------------- 1 file changed, 1 insertion(+), 87 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index c1e28a04f695..f9ab7d9e4ccc 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,92 +41,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.6.1.20001.altinityantalya", { - {"max_insert_threads", 1, 0, "Changed the default from 1 (no parallel execution) to auto (0), which resolves to the number of CPU cores available to the server, reduced under memory pressure via `max_insert_threads_min_free_memory_per_thread`. This parallelizes `INSERT SELECT` by default. Set to 1 to restore the previous single-threaded behavior."}, - {"unique_key_probe_implementation", "auto", "auto", "New setting: selects the UNIQUE KEY probe implementation (currently only the simple baseline exists)"}, - {"s3_base", "", "", "New setting to specify the base URL for resolving relative URLs in the s3 table function and the S3 table engine."}, - {"use_query_condition_cache_for_top_k", false, false, "New setting to gate the query condition cache for `ORDER BY ... LIMIT n` (TopK) reads; disabled by default."}, - {"use_projection_index_in_read_pools", false, false, "New setting to drop mark ranges fully filtered out by a projection index before read tasks are created in MergeTree read pools."}, - {"allow_distinct_partitions_independently", false, true, "New setting to enable independent per-partition evaluation of `DISTINCT` when the partition expression is a deterministic function of the `DISTINCT` columns."}, - {"force_distinct_partitions_independently", false, false, "New setting to force independent per-partition evaluation of `DISTINCT` even when the cost heuristic would skip it."}, - {"max_number_of_partitions_for_independent_distinct", 128, 128, "New setting: maximal number of partitions to apply independent per-partition `DISTINCT`."}, - {"allow_lossy_numeric_supertype", false, false, "New setting that lets if/multiIf/coalesce/ifNull/array/map resolve all-numeric branches with no lossless common type (e.g. Decimal + Float64) to a numeric supertype (Float64, with possible precision loss), so the result can be aggregated. Independent of use_variant_as_common_type: with it off such branches previously raised NO_COMMON_TYPE, with it on they became a Variant; either way they now resolve to Float64."}, - {"throw_on_hive_partitioning_resolution_failure", false, true, "New setting to fail the query when Hive-style partitioning detection for an object storage table cannot list the storage, instead of running without the Hive partition columns."}, - {"allow_experimental_json_ast_dialect", false, false, "New setting to enable the `clickhouse_json` value of the `dialect` setting, which interprets queries as JSON ASTs (the output of `parseQueryToJSON`) instead of SQL text."}, - {"analyzer_compatibility_apply_final_to_all_joined_tables", false, false, "New setting on master (default false = the fixed behavior). The behavior flip itself is recorded under 26.6, and the introduction for backports to older release branches (with default true) under 26.4."}, - {"enable_parallel_single_level_merge", false, true, "New setting to parallelize the final merge of the single-level aggregation hash tables by splitting the key space into disjoint hash partitions that the threads merge independently."}, - {"ai_function_text_default_credentials", "", "", "New setting"}, - {"ai_function_embedding_default_credentials", "", "", "New setting"}, - {"ai_function_allow_insecure_endpoint", true, false, "AI functions now reject insecure (http) endpoints to remote hosts by default."}, - {"ai_function_max_api_calls_per_query", 0, 1000, "Bound outbound AI function HTTP calls per query by default (previously 0 - unlimited)."}, - {"join_runtime_filter_min_probe_rows", 0, 1000, "New setting to control minimum probe side size for installing JOIN runtime filters. It wasn't limited before, so previous value is 0 meaning always install."}, - {"optimize_trivial_count_with_sparsity_filter", false, true, "Promote to BETA and enable by default: serve `SELECT count() FROM t WHERE ` from the persisted per-column `num_defaults` / `num_rows` counters when `` partitions rows into defaults vs non-defaults. Requires the MergeTree setting `compute_exact_num_defaults_for_sparse_columns` (also enabled by default now)."}, - {"input_format_parquet_dictionary_filter_push_down", 0, 1024 * 1024, "New setting enabling Parquet row-group pruning based on dictionary page contents (reader v3). The value is the maximum dictionary page size in bytes for which the optimization applies; 0 (the previous behavior) disables it."}, - {"input_format_read_datetime_number_as_raw_value", true, false, "From 26.8, an unquoted number for a `DateTime`/`DateTime64` column in the `JSON` and `Values`/`Quoted` paths (and in `JSONExtract` and typed `JSON`) is a Unix timestamp in seconds, consistent with the `Values` format, `CAST` and `toDateTime64`. Set this to `true` (or `SET compatibility = '26.7'`) to restore the pre-26.8 behavior, where a bare unquoted integer fed to a `DateTime64` column was read as the raw scaled value (ticks). The tab-separated, CSV and other escaped/whole-text formats are not governed by this setting."}, - {"query_plan_short_circuit_constant_false_join", false, true, "New setting to short-circuit a JOIN with a constant-false ON condition so the non-contributing side is not read. previous_value=false so `compatibility` with versions before 26.8 restores the pre-existing behavior (no short-circuit)."}, - {"input_format_arrow_use_native_reader", true, true, "Obsolete setting, the native ClickHouse reader is now always used for the `Arrow` and `ArrowStream` formats (the Apache Arrow library-based reader has been removed)."}, - {"output_format_arrow_use_native_writer", true, true, "Obsolete setting, the native ClickHouse writer is now always used for the `Arrow` and `ArrowStream` formats (the Apache Arrow library-based writer has been removed)."}, - {"distributed_cache_min_inflight_bytes_to_discard_connection_on_seek", 0, 4 * 1024 * 1024, "New setting to drop and reopen a distributed cache connection on a seek when too many in-flight bytes would otherwise be discarded. Defaults to 4 MiB; 0 restores the previous behavior (always reuse the connection via the read range id)."}, - {"input_format_parquet_spatial_filter_push_down", false, true, "New setting: skip GeoParquet row groups and pages based on spatial predicates and bounding box statistics"}, - {"use_text_index_negative_tokens_cache", false, true, "New setting to cache absent text index tokens and avoid repeated dictionary lookups."}, - }); - addSettingsChanges(settings_changes_history, "26.7", - { - {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, - {"dictionary_lazy_load", "auto", "auto", "New setting overriding the server setting `dictionaries_lazy_load` for an individual dictionary."}, - {"discard_query_data", false, false, "New setting to skip sending query result rows to the client over the native TCP protocol."}, - {"optimize_trivial_count_with_sparsity_filter", false, false, "New (experimental) setting to serve `SELECT count() FROM t WHERE ` from per-column `num_defaults` / `num_rows` recorded in `serialization.json` when `` partitions rows into defaults vs non-defaults."}, - {"merge_tree_generic_exclusion_search_max_steps", 0, 0, "New setting to limit the number of steps of the generic exclusion search over the primary key index."}, - {"use_streaming_marks_compression", false, false, "New setting to compress marks into in-memory representation one block at a time (streaming) instead of materializing the full plain marks array, reducing peak memory during marks loading for compact parts with many substreams."}, - {"s3_validate_etag_on_read", false, true, "New setting to detect concurrent in-place overwrites of S3/GCS objects during a read by validating the GET response ETag against the listed one. previous_value=false so `compatibility` with versions before 26.7 restores the pre-existing behavior (no validation)."}, - {"dead_blobs_to_delay_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, - {"dead_blobs_to_throw_insert", 0, 0, "New setting to override the `MergeTree` setting with the same name per query."}, - {"input_format_csv_missing_nullable_as_empty_string", false, false, "New setting to read a missing value of `Nullable(String)` from CSV as an empty string instead of NULL."}, - {"use_legacy_to_time", true, false, "Use the new `toTime` function (converting values to the `Time` data type) by default instead of the legacy `toTime` (which is still available as `toTimeWithFixedDate`)."}, - {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, - {"parallel_replicas_plan_based", false, false, "New setting"}, - {"use_paimon_metadata_files_cache", false, false, "New setting to enable in-memory caching of parsed Paimon metadata files (manifest lists and manifests). For persistent Paimon table engines it must be enabled before metadata initialization; table functions evaluate it per query. Avoids repeated downloads and deserialization of metadata files from object storage on subsequent queries."}, - {"optimize_or_like_chain", false, true, "Enable by default: optimize OR chains of LIKE/ILIKE/match into multiSearchAny (pure-substring patterns) or multiMatchAny (other patterns, when Hyperscan/Vectorscan is permitted); when neither fast path applies the original OR chain is kept unchanged."}, - {"optimize_or_like_chain_min_patterns", 0, 10, "New setting controlling the minimum number of non-pure-substring LIKE/ILIKE/match branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiMatchAny. Shorter chains are kept as-is because the multiMatchAny (Hyperscan) rewrite only becomes faster than short-circuit OR evaluation from about nine branches."}, - {"optimize_or_like_chain_min_substrings", 0, 4, "New setting controlling the minimum number of pure-substring (%needle%) LIKE/ILIKE branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiSearchAny."}, - {"input_format_arrow_use_native_reader", false, true, "New setting to use the native ClickHouse reader for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, - {"input_format_orc_use_fast_decoder", true, true, "Obsolete setting, the native ClickHouse ORC decoder is now always used (the Apache Arrow-based ORC reader has been removed)."}, - {"output_format_arrow_use_native_writer", false, true, "New setting to use the native ClickHouse writer for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, - {"allow_minmax_index_for_json", true, false, "Forbid creating minmax skip index on JSON columns by default because the index serialization cannot handle heterogeneous Field values"}, - {"s3_allow_server_credentials_in_user_queries", true, false, "New setting to block S3 access from user SQL from resolving the server's own ambient credentials (environment/IMDS/IRSA/instance-profile/AWS-config-file/GCP-OAuth-metadata). Explicit role_arn-based STS assume-role is still allowed. The previous behavior (allowed) is restored with compatibility settings."}, - {"query_plan_merge_expression_into_join", false, true, "New setting. Allow to merge Expression step into JOIN step during join reordering optimization."}, - {"skip_unavailable_shards_mode", "unavailable_or_table_missing", "unavailable_or_table_missing", "New setting to control which exceptions from a remote shard are ignored when `skip_unavailable_shards` is enabled. The default matches the historical behavior: a shard whose table is missing is treated as unavailable."}, - {"use_text_index_tokens_cache", false, true, "Enabled the text index tokens cache globally."}, - {"use_text_index_header_cache", false, true, "Enabled the text index header cache globally."}, - {"optimize_aggregation_in_order_limit", false, true, "New setting to push the `LIMIT` into aggregation-in-order for early termination when the `ORDER BY` is a prefix of the `GROUP BY` sort description."}, - {"explain_query_plan_default", "legacy", "pretty", "From 26.7, `EXPLAIN PLAN` defaults to `actions=1, compact=1, pretty=1`. Set this to `legacy` to restore the pre-26.7 output."}, - {"format_geojson_validate_geometry", true, true, "New setting that controls whether the GeoJSON format enforces RFC 7946 geometry validity (minimum points per line and ring, ring closure, non-empty multi-geometries) when reading and writing"}, - {"use_partition_minmax_for_primary_key_pruning", false, true, "New setting to use the part's partition minmax to prune more granules during primary key analysis for `MergeTree` tables, when a primary key column is also an input column of the partition key."}, - {"allow_delta_lake_writes", false, false, "Added an alias for setting `allow_experimental_delta_lake_writes`, which was moved to Beta."}, - {"allow_experimental_delta_lake_writes", false, false, "Delta Lake writes were moved to Beta."}, - {"optimize_redundant_comparisons", false, true, "New setting to detect conflicting and redundant comparison conditions on the same expression within AND chains."}, - {"mysql_datatypes_support_level", "decimal,datetime64,date2Date32", "decimal,datetime64,date2Date32,geometry", "Map MySQL's concrete spatial types (LINESTRING, POLYGON, MULTILINESTRING, MULTIPOLYGON, MULTIPOINT) and the generic GEOMETRY type to the corresponding ClickHouse geometric types by default. The generic GEOMETRY column maps to the umbrella Geometry type; reading a value whose subtype has no ClickHouse counterpart (GEOMETRYCOLLECTION) throws at read time."}, - {"snappy_mode", "basic", "basic", "New setting to control the wire format used for snappy compression in generic file/URL I/O. The default `basic` preserves backward-compatible Hadoop snappy block format reads; HTTP `Content-Encoding: snappy` always uses the framing format independently of this setting."}, - {"compile_regular_expressions", false, true, "New setting to enable JIT compilation of simple regular expressions in functions like `match` and `extract`."}, - {"min_count_to_compile_regular_expression", 3, 3, "New setting controlling how many times a regular expression must be used before it is JIT-compiled."}, - {"allow_aggregate_partitions_independently", false, true, "Enable independent per-partition aggregation by default when the partition key suits the GROUP BY key. The existing runtime heuristics in `ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePortForAggregation` already skip the optimization when the partition layout is unfavorable (too few partitions, too many partitions, or significantly skewed partition sizes), so enabling the setting is safe in the cases where it would otherwise be a no-op."}, - {"text_index_lazy_intersection_density_threshold", 0.2, 0.2, "Renamed from `text_index_density_threshold` (kept as an alias); selects the posting list intersection algorithm in lazy posting list apply mode."}, - {"allow_experimental_text_index_lazy_apply", false, true, "Lazy posting list apply mode for the text index is no longer experimental; the setting is now obsolete and has no effect (lazy mode is selected via `text_index_posting_list_apply_mode = 'lazy'`)."}, - {"allow_experimental_url_wildcard_from_index_pages", false, false, "New setting to enable expanding wildcards in the `url` table function by listing HTTP index pages."}, - {"url_wildcard_max_directories_to_read", 100000, 100000, "New setting to limit the number of directories read when expanding wildcards in the `url` table function."}, - {"allow_experimental_eval_table_function", false, false, "New setting to enable the experimental table function `eval`."}, - {"output_format_csv_header_serialize_tuple_into_separate_columns", false, true, "New setting. When output_format_csv_serialize_tuple_into_separate_columns is enabled, the CSVWithNames/CSVWithNamesAndTypes header now flattens Tuple columns into their leaf fields so the header width matches the data. Set to false to restore the previous single-name header."}, - {"enable_join_runtime_filters_index_analysis", false, false, "New setting to enable join filtering using dynamic index analysis"}, - {"vector_search_use_quantized_codes", false, false, "New setting to opt into the two-stage approximate vector-search optimization over a Quantize(...) column codec; queries stay exact by default."}, - {"reader_executor_use_long_connections", false, false, "New experimental ReaderExecutor setting (off by default): reuse a held source connection across sequential windows."}, - {"reader_executor_min_bytes_for_seek", 2097152, 2097152, "New experimental ReaderExecutor setting: forward-gap bound for bridging on a held source connection."}, - {"reader_executor_max_tail_for_drain", 1048576, 1048576, "New experimental ReaderExecutor setting: drain bound for completing a dropped long connection."}, - {"precise_float_parsing", false, true, "Use the precise (closest-representable) float parsing algorithm by default, now that it is faster than the previous fast algorithm. Set to false to restore the pre-26.7 fast-but-less-accurate parsing in conversion functions."}, - {"optimize_and_compare_chain_max_hash_work", 0, 5'000'000, "New setting that bounds the work of the `optimize_and_compare_chain` optimization (measured in query-tree nodes hashed) so it cannot dominate analysis of queries with very many or very large `AND`-chains of comparisons. The previous value `0` (unlimited) reproduces the pre-26.7 behavior where the optimization was uncapped, so `compatibility` set to an earlier version keeps deriving transitive predicates without a budget. Set to `0` to disable the budget."}, - {"iceberg_manifest_min_count_to_compact", 30, 30, "New setting to control manifest compaction for Iceberg tables."}, - {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, - {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, - {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, - {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, + {"use_puffin_files_cache", true, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, }); addSettingsChanges(settings_changes_history, "26.6", @@ -235,7 +150,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"allow_experimental_cleanup_old_data_files_compaction", false, false, "New setting"}, {"iceberg_compaction_data_cleanup", 60 * 60 * 3, 60 * 60 * 3, "New setting"}, {"allow_rank_dense_rank_arguments", true, false, "New setting. Before 26.5, the `RANK` and `DENSE_RANK` window functions silently ignored any provided arguments (equivalent to `allow_rank_dense_rank_arguments = 1`). From 26.5, they reject arguments by default with `NUMBER_OF_ARGUMENTS_DOESNT_MATCH` because per SQL standard these functions take zero arguments. Set this to `1` to restore the legacy behavior."}, - {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, }); addSettingsChanges(settings_changes_history, "26.4", { From 13a727894acfae5f85548a12a19d302305de3a91 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 7 Aug 2026 16:39:39 +0200 Subject: [PATCH 6/6] Reuse the `Puffin` format reader instead of a separate `PuffinFile` parser. Related: https://github.com/Altinity/ClickHouse/pull/2179 Related: https://github.com/Altinity/ClickHouse/pull/2183 Co-authored-by: Cursor --- .../Formats/Impl/PuffinBlockInputFormat.cpp | 665 ++++++++++++++++-- .../Formats/Impl/PuffinBlockInputFormat.h | 17 +- .../Iceberg/IcebergDeletionVector.cpp | 1 - .../DataLakes/PuffinDeletionVectorReader.cpp | 148 ++++ .../DataLakes/PuffinDeletionVectorReader.h | 32 + .../ObjectStorage/DataLakes/PuffinFile.cpp | 643 ----------------- .../ObjectStorage/DataLakes/PuffinFile.h | 55 -- ...est_puffin_deletion_vector_footer_bind.cpp | 1 - ...gtest_puffin_non_seekable_buffer_limit.cpp | 3 +- 9 files changed, 821 insertions(+), 744 deletions(-) delete mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp delete mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFile.h diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp index 509cbe93de00..97261e1fd491 100644 --- a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp @@ -1,6 +1,12 @@ +#include #include +#include +#include #include #include +#include +#include +#include #include #include #include @@ -12,15 +18,22 @@ #include #include #include -#include #include #include +#include +#include +#include +#include +#include +#include #include +#include #include +#include #include #include -#include -#include + +#include namespace DB { @@ -28,16 +41,28 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int LZ4_DECODER_FAILED; } namespace { -/// Puffin footer parsing (magic, LZ4 payload, blob metadata JSON) lives in -/// `Storages/ObjectStorage/DataLakes/PuffinFile` and is shared with the Iceberg DV loader. -/// DV blob / materialization ceilings live in `PuffinDeletionVectorReader.h` (`PUFFIN_DV_MAX_*`). - constexpr UInt8 PUFFIN_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; +constexpr UInt8 PUFFIN_FOOTER_COMPRESSED_FLAG = 0x01; +constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; +constexpr size_t PUFFIN_FOOTER_LZ4_MAX_RATIO = 255; +constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; +constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; +constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; +constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; +constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; +constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; +constexpr const char * PUFFIN_DELETION_VECTOR_BLOB_TYPE = "deletion-vector-v1"; + +UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) +{ + return (static_cast(key) << 32) | static_cast(sub_position); +} void checkMagic(const UInt8 * p, const char * context) { @@ -45,6 +70,412 @@ void checkMagic(const UInt8 * p, const char * context) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin magic ({})", context); } +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t blob_region_end, size_t blob_index) +{ + if (offset < static_cast(sizeof(PUFFIN_MAGIC)) || length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); + + Int64 end = 0; + if (common::addOverflow(offset, length, end)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); + + if (static_cast(end) > blob_region_end) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); +} + +void validatePuffinFooterFlags(const UInt8 flags[4]) +{ + if (flags[1] != 0 || flags[2] != 0 || flags[3] != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); + if ((flags[0] & ~PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); +} + +String decompressPuffinFooterPayload(const char * data, size_t size) +{ + LZ4F_dctx * dctx = nullptr; + size_t ret = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to create LZ4 decompression context: {}", LZ4F_getErrorName(ret)); + + struct DecompressionContextGuard + { + LZ4F_dctx * dctx; + explicit DecompressionContextGuard(LZ4F_dctx * dctx_) : dctx(dctx_) { } + ~DecompressionContextGuard() { LZ4F_freeDecompressionContext(dctx); } + }; + DecompressionContextGuard guard(dctx); + + LZ4F_frameInfo_t frame_info{}; + size_t header_size = size; + ret = LZ4F_getFrameInfo(dctx, &frame_info, data, &header_size); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to read LZ4 frame info for Puffin footer: {}", LZ4F_getErrorName(ret)); + + const char * src = data + header_size; + size_t src_remaining = size - header_size; + + size_t max_by_ratio = 0; + if (common::mulOverflow(size, PUFFIN_FOOTER_LZ4_MAX_RATIO, max_by_ratio)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer compressed payload is too large"); + + if (frame_info.contentSize == 0) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame must declare content size"); + + const size_t max_decompressed_size = std::min(max_by_ratio, PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + if (frame_info.contentSize > max_decompressed_size) + { + if (max_decompressed_size == PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 content size {} exceeds absolute decompression limit {}", + frame_info.contentSize, + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 content size {} exceeds decompression limit {}", + frame_info.contentSize, + max_decompressed_size); + } + + String result; + result.resize(static_cast(frame_info.contentSize)); + + size_t dst_offset = 0; + while (true) + { + size_t src_read = src_remaining; + size_t dst_write = result.size() - dst_offset; + ret = LZ4F_decompress(dctx, result.data() + dst_offset, &dst_write, src, &src_read, nullptr); + if (LZ4F_isError(ret)) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to decompress Puffin footer: {}", LZ4F_getErrorName(ret)); + + src += src_read; + src_remaining -= src_read; + dst_offset += dst_write; + + if (ret == 0) + break; + + if (dst_write == 0 && src_read == 0) + throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame is incomplete"); + + if (dst_offset == result.size()) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer decompressed size exceeds declared content size {}", + frame_info.contentSize); + } + + if (dst_offset != frame_info.contentSize) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 decompressed size {} does not match content size {}", + dst_offset, + frame_info.contentSize); + + if (src_remaining != 0) + throw Exception( + ErrorCodes::LZ4_DECODER_FAILED, + "Puffin footer LZ4 frame has {} trailing bytes", + src_remaining); + + result.resize(dst_offset); + return result; +} + +void requireBlobMetadataField(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + if (!blob_obj->has(field_name) || blob_obj->isNull(field_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field '{}'", blob_index, field_name); +} + +bool isJSONInteger(const Poco::Dynamic::Var & value) +{ + return value.isInteger() && !value.isBoolean(); +} + +std::optional tryJSONIntegerAsInt64(const Poco::Dynamic::Var & value) +{ + if (value.isInteger() && !value.isSigned()) + { + const auto as_uint64 = value.convert(); + if (as_uint64 > static_cast(std::numeric_limits::max())) + return std::nullopt; + return static_cast(as_uint64); + } + + return value.convert(); +} + +Int64 requireBlobMetadataInt64(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + requireBlobMetadataField(blob_obj, field_name, blob_index); + const Poco::Dynamic::Var & value = blob_obj->get(field_name); + if (!isJSONInteger(value)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be an integer", blob_index, field_name); + + auto as_int64 = tryJSONIntegerAsInt64(value); + if (!as_int64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' is out of Int64 range", blob_index, field_name); + return *as_int64; +} + +Int32 requireBlobMetadataFieldsElementInt32(const Poco::Dynamic::Var & value, size_t blob_index, size_t field_index) +{ + if (!isJSONInteger(value)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] must be an integer", blob_index, field_index); + + auto as_int64 = tryJSONIntegerAsInt64(value); + if (!as_int64) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int64 range", blob_index, field_index); + + if (*as_int64 < std::numeric_limits::min() || *as_int64 > std::numeric_limits::max()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int32 range", blob_index, field_index); + + return static_cast(*as_int64); +} + +String requireJSONStringValue(const Poco::Dynamic::Var & value, size_t blob_index, const char * field_name) +{ + if (!value.isString()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); + return value.extract(); +} + +String requireBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) +{ + requireBlobMetadataField(blob_obj, field_name, blob_index); + return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); +} + +void requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) +{ + static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; + for (const char * key : required_properties) + { + auto it = blob.properties.find(key); + if (it == blob.properties.end() || it->second.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property '{}'", + blob_index, + key); + } + + UInt64 cardinality = 0; + if (!tryParse(cardinality, blob.properties.at("cardinality"))) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + blob_index); +} + +void parseStringValuedProperties( + const Poco::JSON::Object::Ptr & props_obj, + std::map * out, + bool for_blob, + size_t blob_index) +{ + for (const auto & [key, val] : *props_obj) + { + if (!val.isString()) + { + if (for_blob) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: property '{}' must be a string", blob_index, key); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer property '{}' must be a string", key); + } + if (out) + out->emplace(key, val.extract()); + } +} + +void parseBlobProperties(const Poco::JSON::Object::Ptr & blob_obj, PuffinBlob & blob, size_t blob_index, bool required) +{ + if (!blob_obj->has("properties")) + { + if (required) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field 'properties'", blob_index); + return; + } + + if (blob_obj->isNull("properties")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); + + auto props_obj = blob_obj->getObject("properties"); + if (!props_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); + + parseStringValuedProperties(props_obj, &blob.properties, /*for_blob=*/true, blob_index); +} + +void validateFileMetadataProperties(const Poco::JSON::Object::Ptr & footer_obj) +{ + if (!footer_obj->has("properties")) + return; + + if (footer_obj->isNull("properties")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); + + auto props_obj = footer_obj->getObject("properties"); + if (!props_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); + + parseStringValuedProperties(props_obj, /*out=*/nullptr, /*for_blob=*/false, /*blob_index=*/0); +} + +std::vector parseFooterJSON(const String & footer_json, size_t blob_region_end) +{ + Poco::JSON::Parser parser; + Poco::Dynamic::Var root; + try + { + root = parser.parse(footer_json); + } + catch (const Exception &) + { + throw; + } + catch (const Poco::Exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Puffin footer JSON: {}", e.displayText()); + } + + if (root.type() != typeid(Poco::JSON::Object::Ptr)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer JSON must be an object"); + + auto obj = root.extract(); + + validateFileMetadataProperties(obj); + + if (!obj->has("blobs") || obj->isNull("blobs")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer is missing required field 'blobs'"); + + auto blobs_arr = obj->getArray("blobs"); + if (!blobs_arr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'blobs' must be an array"); + + std::vector blobs; + for (size_t i = 0; i < blobs_arr->size(); ++i) + { + auto blob_obj = blobs_arr->getObject(static_cast(i)); + if (!blob_obj) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: must be an object", i); + + PuffinBlob blob; + + blob.type = requireBlobMetadataString(blob_obj, "type", i); + blob.snapshot_id = requireBlobMetadataInt64(blob_obj, "snapshot-id", i); + blob.sequence_number = requireBlobMetadataInt64(blob_obj, "sequence-number", i); + blob.offset = requireBlobMetadataInt64(blob_obj, "offset", i); + blob.length = requireBlobMetadataInt64(blob_obj, "length", i); + + if (blob.type == PUFFIN_DELETION_VECTOR_BLOB_TYPE) + { + if (blob_obj->has("compression-codec")) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 must omit 'compression-codec'", + i); + + parseBlobProperties(blob_obj, blob, i, /*required=*/true); + requireDeletionVectorV1Properties(blob, i); + } + else + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: unsupported blob type '{}', only '{}' is supported", + i, + blob.type, + PUFFIN_DELETION_VECTOR_BLOB_TYPE); + } + + requireBlobMetadataField(blob_obj, "fields", i); + auto fields_arr = blob_obj->getArray("fields"); + if (!fields_arr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'fields' must be an array", i); + + for (size_t j = 0; j < fields_arr->size(); ++j) + blob.fields.push_back(requireBlobMetadataFieldsElementInt32(fields_arr->get(static_cast(j)), i, j)); + + validatePuffinBlobBounds(blob.offset, blob.length, blob_region_end, i); + + blobs.push_back(std::move(blob)); + } + return blobs; +} + +std::vector readPuffinFooterFromSeekableImpl(SeekableReadBuffer & seekable, size_t file_size) +{ + if (file_size < 16) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); + + seekable.seek(0, SEEK_SET); + char magic_buf[4]; + seekable.readStrict(magic_buf, 4); + checkMagic(reinterpret_cast(magic_buf), "header"); + + seekable.seek(static_cast(file_size - PUFFIN_FOOTER_TRAILER_SIZE), SEEK_SET); + Int32 footer_length_signed = 0; + readBinaryLittleEndian(footer_length_signed, seekable); + + UInt8 flags[4] = {}; + seekable.readStrict(reinterpret_cast(flags), sizeof(flags)); + + char trailing_buf[4]; + seekable.readStrict(trailing_buf, 4); + checkMagic(reinterpret_cast(trailing_buf), "trailing"); + validatePuffinFooterFlags(flags); + + if (footer_length_signed <= 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + UInt64 footer_trailer_size = 0; + if (common::addOverflow( + static_cast(footer_length_signed), + static_cast(PUFFIN_FOOTER_TRAILER_SIZE), + footer_trailer_size) + || footer_trailer_size > file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + const size_t footer_length = static_cast(footer_length_signed); + const size_t payload_start = file_size - PUFFIN_FOOTER_TRAILER_SIZE - footer_length; + /// Footer layout is Magic | FooterPayload | …; require a distinct footer-open Magic after the + /// file header Magic (payload_start == 4 would reuse the header Magic when blobs is empty). + if (payload_start < 2 * sizeof(PUFFIN_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); + + if (footer_length > PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin footer payload size {} exceeds absolute limit {}", + footer_length, + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); + + seekable.seek(static_cast(payload_start - sizeof(PUFFIN_MAGIC)), SEEK_SET); + char leading_magic_buf[4]; + seekable.readStrict(leading_magic_buf, 4); + checkMagic(reinterpret_cast(leading_magic_buf), "footer"); + + String footer_payload(footer_length, '\0'); + seekable.seek(static_cast(payload_start), SEEK_SET); + seekable.readStrict(footer_payload.data(), footer_length); + + String footer_json; + if ((flags[0] & PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) + footer_json = decompressPuffinFooterPayload(footer_payload.data(), footer_payload.size()); + else + footer_json = std::move(footer_payload); + + const size_t blob_region_end = payload_start - sizeof(PUFFIN_MAGIC); + return parseFooterJSON(footer_json, blob_region_end); +} + PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) { PuffinFooter result; @@ -52,27 +483,27 @@ PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) auto * seekable = dynamic_cast(&buf); auto file_size_opt = tryGetFileSizeFromReadBuffer(buf); - /// Pipes/FIFOs are SeekableReadBuffer subclasses but fstat reports size 0; require a real - /// regular file before trusting seek+size (same pattern as ORC/Arrow). Also honor - /// `input_format_allow_seeks` via FormatSettings.seekable_read. if (seekable_read && seekable && seekable->checkIfActuallySeekable() && file_size_opt) { - result.blobs = readPuffinFooterBlobsFromSeekable(*seekable, *file_size_opt); + result.blobs = readPuffinFooterFromSeekableImpl(*seekable, *file_size_opt); } else { - /// Fail fast on format mismatch before buffering the rest of a (possibly huge) stream — - /// same pattern as `asArrowFileLoadIntoMemory`. result.data.resize(sizeof(PUFFIN_MAGIC)); const size_t magic_read = buf.read(reinterpret_cast(result.data.data()), sizeof(PUFFIN_MAGIC)); if (magic_read < sizeof(PUFFIN_MAGIC)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); checkMagic(result.data.data(), "header"); - appendReadBufferWithAbsoluteSizeLimit(buf, result.data, PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE); + std::vector tmp(DEFAULT_BLOCK_SIZE); + while (!buf.eof()) + { + size_t n = buf.read(reinterpret_cast(tmp.data()), tmp.size()); + result.data.insert(result.data.end(), tmp.data(), tmp.data() + n); + } ReadBufferFromMemory mem_buf(result.data.data(), result.data.size()); - result.blobs = readPuffinFooterBlobsFromSeekable(mem_buf, result.data.size()); + result.blobs = readPuffinFooterFromSeekableImpl(mem_buf, result.data.size()); } return result; @@ -122,10 +553,6 @@ void readDeletionVectorEnvelopePrefix( seekable->readStrict(reinterpret_cast(header), 8); } -/// Validate absolute size and the first 8 envelope bytes (combined_length + magic) before allocating -/// `blob.length`. CRC / roaring deserialize run later in `deserializeDeletionVectorV1Blob` after a -/// bounded read — we intentionally do not stream roaring here (larger redesign; CRC coverage is -/// over magic+vector). String readDeletionVectorBlobBytes( const PuffinBlob & blob, ReadBuffer & buf, const std::vector & data, bool seekable_read) { @@ -144,11 +571,168 @@ String readDeletionVectorBlobBytes( UInt8 header[8]; readDeletionVectorEnvelopePrefix(blob, buf, data, seekable_read, header); - validateDeletionVectorEnvelope(header, blob.length); + + ReadBufferFromMemory header_buf(reinterpret_cast(header), sizeof(header)); + UInt32 combined_length = 0; + readBinaryBigEndian(combined_length, header_buf); + if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + UInt64 expected_blob_size = 0; + if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + if (static_cast(blob.length) != expected_blob_size) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob size {} does not match combined length {}", + blob.length, + combined_length); return readPuffinBlobBytes(blob, buf, data, seekable_read); } +roaring::Roaring readRoaringPortableSafe(const char * data, size_t size, Int32 key) +{ + try + { + return roaring::Roaring::readSafe(data, size); + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to deserialize deletion vector roaring bitmap at key {}: {}", key, e.what()); + } +} + +void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_cardinality, ColumnUInt64 & positions) +{ + if (bytes.size() < sizeof(Int64)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is too small"); + + const char * ptr = bytes.data(); + size_t remaining = bytes.size(); + + /// Iceberg deletion-vector roaring layout stores count and keys as little-endian. + const Int64 bitmap_count = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int64); + remaining -= sizeof(Int64); + + if (bitmap_count < 0 || bitmap_count > std::numeric_limits::max()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap count: {}", bitmap_count); + + Int32 last_key = -1; + Int32 remaining_count = static_cast(bitmap_count); + UInt64 running_cardinality = 0; + + while (remaining_count > 0) + { + if (remaining < sizeof(Int32)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is truncated while reading key"); + + const Int32 key = unalignedLoadLittleEndian(ptr); + ptr += sizeof(Int32); + remaining -= sizeof(Int32); + + if (key < 0 || key > DELETION_VECTOR_MAX_KEY) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap key: {}", key); + if (key <= last_key) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap keys must be sorted in ascending order"); + + auto bitmap = readRoaringPortableSafe(ptr, remaining, key); + + const size_t bitmap_size = bitmap.getSizeInBytes(/*portable=*/true); + if (bitmap_size > remaining) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector roaring bitmap at key {} exceeds blob size", key); + + const UInt64 bitmap_cardinality = bitmap.cardinality(); + UInt64 new_running_cardinality = 0; + if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality exceeds declared cardinality {}", + expected_cardinality); + + if (new_running_cardinality > expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds declared cardinality {}", + new_running_cardinality, + expected_cardinality); + + running_cardinality = new_running_cardinality; + + for (UInt32 sub_position : bitmap) + { + const UInt64 position = positionFromKeyAndSubPosition(static_cast(key), sub_position); + if (position > static_cast(DELETION_VECTOR_MAX_POSITION)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector position {} is out of supported range", position); + positions.insertValue(position); + } + + ptr += bitmap_size; + remaining -= bitmap_size; + last_key = key; + --remaining_count; + } + + if (remaining != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap has {} trailing bytes", remaining); + + if (running_cardinality != expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} does not match deserialized row count {}", + expected_cardinality, + running_cardinality); +} + +std::string_view extractDeletionVectorPayload(std::string_view blob) +{ + if (blob.size() < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + + const auto * blob_bytes = reinterpret_cast(blob.data()); + ReadBufferFromMemory mem_buf(blob.data(), blob.size()); + UInt32 combined_length = 0; + readBinaryBigEndian(combined_length, mem_buf); + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + const size_t vector_size = combined_length - sizeof(DELETION_VECTOR_MAGIC); + const size_t expected_blob_size = sizeof(UInt32) + combined_length + sizeof(UInt32); + if (blob.size() != expected_blob_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob size {} does not match combined length {}", blob.size(), combined_length); + + if (std::memcmp(blob_bytes + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + const UInt8 * crc_input = blob_bytes + sizeof(UInt32); + const size_t crc_input_size = combined_length; + mem_buf.ignore(combined_length); + UInt32 expected_crc = 0; + readBinaryBigEndian(expected_crc, mem_buf); + const UInt32 actual_crc = static_cast(crc32_z(0L, reinterpret_cast(crc_input), crc_input_size)); + if (expected_crc != actual_crc) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector CRC mismatch"); + + return std::string_view(blob.data() + 2 * sizeof(UInt32), vector_size); +} + +void deserializeDeletionVectorV1(std::string_view blob, UInt64 expected_cardinality, ColumnUInt64 & positions) +{ + if (expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + deserializeRoaringPositionBitmap(extractDeletionVectorPayload(blob), expected_cardinality, positions); +} + NamesAndTypesList getPuffinMetadataSchema() { return { @@ -218,6 +802,11 @@ void checkPuffinHeader(const Block & header) } +std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size) +{ + return readPuffinFooterFromSeekableImpl(seekable, file_size); +} + PuffinMetadataInputFormat::PuffinMetadataInputFormat(ReadBuffer & buf, SharedHeader header_, const FormatSettings & format_settings_) : IInputFormat(std::move(header_), &buf) , seekable_read(format_settings_.seekable_read) @@ -231,7 +820,6 @@ Chunk PuffinMetadataInputFormat::read() { blob_index = 0; footer = readPuffinFooter(*in, seekable_read); - /// Metadata never reads blob payloads; drop the full-file buffer from the non-seekable path. footer.data.clear(); footer.data.shrink_to_fit(); initialized = true; @@ -309,7 +897,6 @@ Chunk PuffinInputFormat::read() { blob_index = 0; footer = readPuffinFooter(*in, seekable_read); - /// No deletion-vector payload will be read; drop the full-file buffer from the non-seekable path. if (!need_deleted_rows) { footer.data.clear(); @@ -323,8 +910,13 @@ Chunk PuffinInputFormat::read() const size_t current_blob_index = blob_index; const auto & blob = footer.blobs[blob_index++]; - if (blob.type != "deletion-vector-v1") - continue; + if (blob.type != PUFFIN_DELETION_VECTOR_BLOB_TYPE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: unexpected blob type '{}', only '{}' is supported", + current_blob_index, + blob.type, + PUFFIN_DELETION_VECTOR_BLOB_TYPE); const auto & referenced_data_file = blob.properties.at("referenced-data-file"); @@ -339,24 +931,11 @@ Chunk PuffinInputFormat::read() col_file->insertData(referenced_data_file.data(), referenced_data_file.size()); MutableColumnPtr col_rows; - // Payload I/O and validation (envelope / CRC / roaring) run only when `deleted_rows` is - // projected. Footer property checks above stay projection-independent. - // Intentional: subset reads of `referenced_data_file` must not force reading up to - // PUFFIN_DV_MAX_BLOB_SIZE for blobs that will never be materialized. - // Materialization ceiling uses footer `cardinality` and must run before blob allocate. if (need_deleted_rows) { - if (expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality {} exceeds materialization limit {}", - expected_cardinality, - PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); - const String blob_data = readDeletionVectorBlobBytes(blob, *in, footer.data, seekable_read); - const auto positions = deserializeDeletionVectorV1Blob(blob_data, expected_cardinality); auto col_rows_data = ColumnUInt64::create(); - col_rows_data->getData().insert(positions.begin(), positions.end()); + deserializeDeletionVectorV1(blob_data, expected_cardinality, *col_rows_data); auto col_rows_offsets = ColumnArray::ColumnOffsets::create(); col_rows_offsets->insertValue(col_rows_data->size()); @@ -430,6 +1009,8 @@ void registerInputFormatPuffin(FormatFactory & factory) Special input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) file footer metadata. It outputs one row per blob entry from the footer `BlobMetadata` list. +`deletion-vector-v1` is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. + Fixed output columns: - `blob_type` (`String`) - blob type, for example `deletion-vector-v1` - `snapshot_id` (`Int64`) - snapshot id of the blob @@ -442,7 +1023,7 @@ Fixed output columns: Optional top-level `FileMetadata.properties` in the footer (for example `created-by`) are type-checked when present but are not returned as columns. If the key is present it must be a JSON object with string values (null is rejected). -LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. Non-seekable input (pipes / `input_format_allow_seeks = 0`) buffers the file up to an absolute ceiling sized for one max deletion-vector blob plus max footer; larger inputs require seekable reads. +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. ## Example usage {#example-usage} @@ -471,18 +1052,18 @@ Pair with the `Puffin` format to read `deletion-vector-v1` blob payloads. Input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) files. -The format exposes deleted row positions from `deletion-vector-v1` blobs. Other blob types (for example `apache-datasketches-theta-v1`) are skipped. +The format exposes deleted row positions from `deletion-vector-v1` blobs. It is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. If a puffin file contains multiple `deletion-vector-v1` blobs, the format outputs one row per such blob. Fixed output columns: - `referenced_data_file` (`String`) - location of the data file the deletion vector applies to (`referenced-data-file` blob property) - `deleted_rows` (`Array(UInt64)`) - 64-bit row positions deleted according to the deletion vector roaring bitmap -Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested, before the blob is allocated. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. +Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. -LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. Non-seekable input (pipes / `input_format_allow_seeks = 0`) buffers the file up to an absolute ceiling sized for one max deletion-vector blob plus max footer; larger inputs require seekable reads. +LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. Only a subset of output columns can be requested. A user-provided structure with unexpected column names or types is rejected when the format is created. diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h index cf2cf5e1e5d3..37d046a4cbe7 100644 --- a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h @@ -1,20 +1,35 @@ #pragma once #include +#include #include #include -#include namespace DB { +struct PuffinBlob +{ + String type; + Int64 snapshot_id = 0; + Int64 sequence_number = 0; + std::vector fields; + Int64 offset = 0; + Int64 length = 0; + String compression_codec; + std::map properties; +}; + struct PuffinFooter { std::vector blobs; std::vector data; }; +/// Shared with the Iceberg deletion-vector loader (seekable object-storage path). +std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size); + class PuffinMetadataInputFormat : public IInputFormat { public: diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp index 892945961a50..eaaf9a0cdaa5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp index 834ee6d42f6e..f66e8e836b98 100644 --- a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -303,4 +304,151 @@ std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset return deserializeDeletionVectorV1Blob(blob_data, expected_cardinality); } +void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size) +{ + if (out.size() > max_buffered_size) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable buffer size {} exceeds absolute limit {}", + out.size(), + max_buffered_size); + } + + std::vector tmp(DBMS_DEFAULT_BUFFER_SIZE); + while (!buf.eof()) + { + const size_t capacity = max_buffered_size - out.size(); + if (capacity == 0) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", + max_buffered_size); + } + + const size_t to_read = std::min(tmp.size(), capacity); + const size_t n = buf.read(reinterpret_cast(tmp.data()), to_read); + if (n == 0) + break; + + out.insert(out.end(), tmp.data(), tmp.data() + n); + + /// If we filled the remaining capacity and the stream still has data, fail closed. + if (n == capacity && !buf.eof()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", + max_buffered_size); + } + } +} + +const PuffinBlob & bindDeletionVectorBlob( + const std::vector & blobs, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view expected_referenced_data_file, + UInt64 expected_cardinality) +{ + const PuffinBlob * matched = nullptr; + size_t matched_index = 0; + + for (size_t i = 0; i < blobs.size(); ++i) + { + if (blobs[i].offset != content_offset || blobs[i].length != content_size_in_bytes) + continue; + + if (matched) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Multiple Puffin blobs claim offset {} length {}", + content_offset, + content_size_in_bytes); + } + + matched = &blobs[i]; + matched_index = i; + } + + if (!matched) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "No Puffin footer blob at offset {} length {}", + content_offset, + content_size_in_bytes); + } + + if (matched->type != "deletion-vector-v1") + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} at offset {} length {} has type '{}', expected deletion-vector-v1", + matched_index, + content_offset, + content_size_in_bytes, + matched->type); + } + + if (!matched->compression_codec.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 must omit compression-codec", + matched_index); + } + + const auto ref_it = matched->properties.find("referenced-data-file"); + if (ref_it == matched->properties.end() || ref_it->second.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property 'referenced-data-file'", + matched_index); + } + + if (ref_it->second != expected_referenced_data_file) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} referenced-data-file '{}' does not match expected data file '{}'", + matched_index, + ref_it->second, + expected_referenced_data_file); + } + + const auto card_it = matched->properties.find("cardinality"); + if (card_it == matched->properties.end() || card_it->second.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property 'cardinality'", + matched_index); + } + + UInt64 footer_cardinality = 0; + if (!tryParse(footer_cardinality, card_it->second)) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + matched_index); + } + + if (footer_cardinality != expected_cardinality) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} cardinality {} does not match expected cardinality {}", + matched_index, + footer_cardinality, + expected_cardinality); + } + + return *matched; +} + } diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h index e025a7d39675..d3742b26123d 100644 --- a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -16,6 +17,18 @@ constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; /// Absolute cap on materialized deleted positions (~800 MiB of UInt64s at this limit). constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; +/// Leading / footer-open magic is always 4 bytes (`PFA1`). +constexpr size_t PUFFIN_MAGIC_SIZE = 4; +constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; +/// Absolute cap on footer payload size (uncompressed JSON bytes, or declared LZ4 contentSize). +constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; + +/// Non-seekable SQL path must buffer the whole file to reach the trailer. Cap total buffered size +/// so a crafted pipe cannot allocate unbounded memory before footer-length validation. +/// Sized for one max DV blob + max footer (header magic + blob + footer magic + payload + trailer). +constexpr size_t PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE = PUFFIN_MAGIC_SIZE + PUFFIN_DV_MAX_BLOB_SIZE + + PUFFIN_MAGIC_SIZE + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE + PUFFIN_FOOTER_TRAILER_SIZE; + /// Validate that [offset, offset + length) fits within file_size. void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context = "Puffin deletion vector"); @@ -29,4 +42,23 @@ std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, /// Read a deletion-vector-v1 blob from a Puffin file at the given offset and length. std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality = std::nullopt); +/// Append bytes from `buf` into `out` until EOF. Throws if `out` would exceed `max_buffered_size`. +void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size); + +/// Compatibility alias for Iceberg / tests: same as `readPuffinFooterFromSeekable`. +inline std::vector readPuffinFooterBlobsFromSeekable(SeekableReadBuffer & seekable, size_t file_size) +{ + return readPuffinFooterFromSeekable(seekable, file_size); +} + +/// Find the unique footer blob at (`content_offset`, `content_size_in_bytes`) and bind it as a +/// deletion-vector-v1 for `expected_referenced_data_file` with `expected_cardinality`. +/// Throws if the slice is missing, ambiguous, or does not match the expected DV identity. +const PuffinBlob & bindDeletionVectorBlob( + const std::vector & blobs, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view expected_referenced_data_file, + UInt64 expected_cardinality); + } diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp deleted file mode 100644 index 728937db4ae2..000000000000 --- a/src/Storages/ObjectStorage/DataLakes/PuffinFile.cpp +++ /dev/null @@ -1,643 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include - -namespace ProfileEvents -{ -extern const Event PuffinFilesRead; -extern const Event PuffinFileReadMicroseconds; -} - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int LZ4_DECODER_FAILED; -} - -namespace -{ - -struct ScopedPuffinFileReadProfileEvent -{ - ProfileEventTimeIncrement watch; - - ScopedPuffinFileReadProfileEvent() - : watch(ProfileEvents::PuffinFileReadMicroseconds) - { - ProfileEvents::increment(ProfileEvents::PuffinFilesRead); - } -}; - -constexpr UInt8 PUFFIN_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; -static_assert(sizeof(PUFFIN_MAGIC) == PUFFIN_MAGIC_SIZE); -constexpr UInt8 PUFFIN_FOOTER_COMPRESSED_FLAG = 0x01; -constexpr size_t PUFFIN_FOOTER_LZ4_MAX_RATIO = 255; -/// DV blob / materialization ceilings live in `PuffinDeletionVectorReader.h` (`PUFFIN_DV_MAX_*`) -/// and are shared with the Iceberg deletion-vector reader. Applied only when `deleted_rows` is -/// requested; subset reads that skip materialization do not enforce the materialization ceiling. -/// Intentionally not FormatSettings knobs: fail-closed amplification guards. - -void checkMagic(const UInt8 * p, const char * context) -{ - if (std::memcmp(p, PUFFIN_MAGIC, 4) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin magic ({})", context); -} - -/// `blob_region_end` is the end of the `Blob₁ ... Blobₙ` region, i.e. the offset of the footer's -/// leading magic. Blob offset/length must stay within [PUFFIN_MAGIC, blob_region_end), so a malformed -/// footer cannot point a blob into the footer payload or trailer. -void validatePuffinFooterBlobBounds(Int64 offset, Int64 length, size_t blob_region_end, size_t blob_index) -{ - if (offset < static_cast(sizeof(PUFFIN_MAGIC)) || length < 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); - - Int64 end = 0; - if (common::addOverflow(offset, length, end)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); - - if (static_cast(end) > blob_region_end) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: offset/length out of bounds", blob_index); -} - -void validatePuffinFooterFlags(const UInt8 flags[4]) -{ - if (flags[1] != 0 || flags[2] != 0 || flags[3] != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); - if ((flags[0] & ~PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Puffin footer flags"); -} - -String decompressPuffinFooterPayload(const char * data, size_t size) -{ - LZ4F_dctx * dctx = nullptr; - size_t ret = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); - if (LZ4F_isError(ret)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to create LZ4 decompression context: {}", LZ4F_getErrorName(ret)); - - struct DecompressionContextGuard - { - LZ4F_dctx * dctx; - explicit DecompressionContextGuard(LZ4F_dctx * dctx_) : dctx(dctx_) { } - ~DecompressionContextGuard() { LZ4F_freeDecompressionContext(dctx); } - }; - DecompressionContextGuard guard(dctx); - - LZ4F_frameInfo_t frame_info{}; - size_t header_size = size; - ret = LZ4F_getFrameInfo(dctx, &frame_info, data, &header_size); - if (LZ4F_isError(ret)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to read LZ4 frame info for Puffin footer: {}", LZ4F_getErrorName(ret)); - - const char * src = data + header_size; - size_t src_remaining = size - header_size; - - size_t max_by_ratio = 0; - if (common::mulOverflow(size, PUFFIN_FOOTER_LZ4_MAX_RATIO, max_by_ratio)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer compressed payload is too large"); - - if (frame_info.contentSize == 0) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame must declare content size"); - - const size_t max_decompressed_size = std::min(max_by_ratio, PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); - if (frame_info.contentSize > max_decompressed_size) - { - if (max_decompressed_size == PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 content size {} exceeds absolute decompression limit {}", - frame_info.contentSize, - PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); - - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 content size {} exceeds decompression limit {}", - frame_info.contentSize, - max_decompressed_size); - } - - String result; - result.resize(static_cast(frame_info.contentSize)); - - size_t dst_offset = 0; - while (true) - { - size_t src_read = src_remaining; - size_t dst_write = result.size() - dst_offset; - ret = LZ4F_decompress(dctx, result.data() + dst_offset, &dst_write, src, &src_read, nullptr); - if (LZ4F_isError(ret)) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Failed to decompress Puffin footer: {}", LZ4F_getErrorName(ret)); - - src += src_read; - src_remaining -= src_read; - dst_offset += dst_write; - - if (ret == 0) - break; - - /// No forward progress while the frame is still open means the input is truncated: - /// LZ4F_decompress wants more input but we have given it everything. Stop to avoid spinning. - if (dst_write == 0 && src_read == 0) - throw Exception(ErrorCodes::LZ4_DECODER_FAILED, "Puffin footer LZ4 frame is incomplete"); - - if (dst_offset == result.size()) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer decompressed size exceeds declared content size {}", - frame_info.contentSize); - } - - if (dst_offset != frame_info.contentSize) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 decompressed size {} does not match content size {}", - dst_offset, - frame_info.contentSize); - - if (src_remaining != 0) - throw Exception( - ErrorCodes::LZ4_DECODER_FAILED, - "Puffin footer LZ4 frame has {} trailing bytes", - src_remaining); - - result.resize(dst_offset); - return result; -} - -void requireBlobMetadataField(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - if (!blob_obj->has(field_name) || blob_obj->isNull(field_name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field '{}'", blob_index, field_name); -} - -/// Poco reports JSON booleans as integers (`std::numeric_limits::is_integer`). -bool isJSONInteger(const Poco::Dynamic::Var & value) -{ - return value.isInteger() && !value.isBoolean(); -} - -/// Poco stores JSON integers that do not fit signed Int64 as unsigned; convert() would throw RangeException. -std::optional tryJSONIntegerAsInt64(const Poco::Dynamic::Var & value) -{ - if (value.isInteger() && !value.isSigned()) - { - const auto as_uint64 = value.convert(); - if (as_uint64 > static_cast(std::numeric_limits::max())) - return std::nullopt; - return static_cast(as_uint64); - } - - return value.convert(); -} - -Int64 requireBlobMetadataInt64(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - requireBlobMetadataField(blob_obj, field_name, blob_index); - const Poco::Dynamic::Var & value = blob_obj->get(field_name); - if (!isJSONInteger(value)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be an integer", blob_index, field_name); - - auto as_int64 = tryJSONIntegerAsInt64(value); - if (!as_int64) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' is out of Int64 range", blob_index, field_name); - return *as_int64; -} - -Int32 requireBlobMetadataFieldsElementInt32(const Poco::Dynamic::Var & value, size_t blob_index, size_t field_index) -{ - if (!isJSONInteger(value)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] must be an integer", blob_index, field_index); - - auto as_int64 = tryJSONIntegerAsInt64(value); - if (!as_int64) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int64 range", blob_index, field_index); - - if (*as_int64 < std::numeric_limits::min() || *as_int64 > std::numeric_limits::max()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: fields[{}] is out of Int32 range", blob_index, field_index); - - return static_cast(*as_int64); -} - -String requireJSONStringValue(const Poco::Dynamic::Var & value, size_t blob_index, const char * field_name) -{ - if (!value.isString()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); - return value.extract(); -} - -String requireBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - requireBlobMetadataField(blob_obj, field_name, blob_index); - return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); -} - -String optBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const char * field_name, size_t blob_index) -{ - if (!blob_obj->has(field_name)) - return {}; - if (blob_obj->isNull(field_name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field '{}' must be a string", blob_index, field_name); - return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); -} - -void requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) -{ - static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; - for (const char * key : required_properties) - { - auto it = blob.properties.find(key); - if (it == blob.properties.end() || it->second.empty()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 missing required property '{}'", - blob_index, - key); - } - - UInt64 cardinality = 0; - if (!tryParse(cardinality, blob.properties.at("cardinality"))) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", - blob_index); -} - -void parseStringValuedProperties( - const Poco::JSON::Object::Ptr & props_obj, - std::map * out, - bool for_blob, - size_t blob_index) -{ - for (const auto & [key, val] : *props_obj) - { - if (!val.isString()) - { - if (for_blob) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: property '{}' must be a string", blob_index, key); - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer property '{}' must be a string", key); - } - if (out) - out->emplace(key, val.extract()); - } -} - -void parseBlobProperties(const Poco::JSON::Object::Ptr & blob_obj, PuffinBlob & blob, size_t blob_index, bool required) -{ - if (!blob_obj->has("properties")) - { - if (required) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: missing required field 'properties'", blob_index); - return; - } - - if (blob_obj->isNull("properties")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); - - auto props_obj = blob_obj->getObject("properties"); - if (!props_obj) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'properties' must be an object", blob_index); - - parseStringValuedProperties(props_obj, &blob.properties, /*for_blob=*/true, blob_index); -} - -/// Optional FileMetadata.properties: JSON object with string values (Iceberg Puffin spec). -/// Absent is allowed; present null / non-object is rejected. -void validateFileMetadataProperties(const Poco::JSON::Object::Ptr & footer_obj) -{ - if (!footer_obj->has("properties")) - return; - - if (footer_obj->isNull("properties")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); - - auto props_obj = footer_obj->getObject("properties"); - if (!props_obj) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'properties' must be an object"); - - parseStringValuedProperties(props_obj, /*out=*/nullptr, /*for_blob=*/false, /*blob_index=*/0); -} - -std::vector parseFooterJSON(const String & footer_json, size_t blob_region_end) -{ - Poco::JSON::Parser parser; - Poco::Dynamic::Var root; - try - { - root = parser.parse(footer_json); - } - catch (const Exception &) - { - /// catch `MEMORY_LIMIT_EXCEEDED` and other possible exceptions from `MemoryTracker` - throw; - } - catch (const Poco::Exception & e) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Puffin footer JSON: {}", e.displayText()); - } - - if (root.type() != typeid(Poco::JSON::Object::Ptr)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer JSON must be an object"); - - auto obj = root.extract(); - - validateFileMetadataProperties(obj); - - if (!obj->has("blobs") || obj->isNull("blobs")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer is missing required field 'blobs'"); - - auto blobs_arr = obj->getArray("blobs"); - if (!blobs_arr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin footer field 'blobs' must be an array"); - - std::vector blobs; - for (size_t i = 0; i < blobs_arr->size(); ++i) - { - auto blob_obj = blobs_arr->getObject(static_cast(i)); - if (!blob_obj) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: must be an object", i); - - PuffinBlob blob; - - blob.type = requireBlobMetadataString(blob_obj, "type", i); - blob.snapshot_id = requireBlobMetadataInt64(blob_obj, "snapshot-id", i); - blob.sequence_number = requireBlobMetadataInt64(blob_obj, "sequence-number", i); - blob.offset = requireBlobMetadataInt64(blob_obj, "offset", i); - blob.length = requireBlobMetadataInt64(blob_obj, "length", i); - - if (blob.type == "deletion-vector-v1") - { - if (blob.snapshot_id != -1 || blob.sequence_number != -1) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 snapshot-id and sequence-number must be -1", - i); - - /// Spec requires the key to be omitted; present null is not omission. - if (blob_obj->has("compression-codec")) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 must omit 'compression-codec'", - i); - - parseBlobProperties(blob_obj, blob, i, /*required=*/true); - requireDeletionVectorV1Properties(blob, i); - } - else - { - blob.compression_codec = optBlobMetadataString(blob_obj, "compression-codec", i); - parseBlobProperties(blob_obj, blob, i, /*required=*/false); - } - - requireBlobMetadataField(blob_obj, "fields", i); - auto fields_arr = blob_obj->getArray("fields"); - if (!fields_arr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin blob {}: field 'fields' must be an array", i); - - for (size_t j = 0; j < fields_arr->size(); ++j) - blob.fields.push_back(requireBlobMetadataFieldsElementInt32(fields_arr->get(static_cast(j)), i, j)); - - validatePuffinFooterBlobBounds(blob.offset, blob.length, blob_region_end, i); - - blobs.push_back(std::move(blob)); - } - return blobs; -} - -} // namespace - -void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size) -{ - if (out.size() > max_buffered_size) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin non-seekable buffer size {} exceeds absolute limit {}", - out.size(), - max_buffered_size); - } - - std::vector tmp(DBMS_DEFAULT_BUFFER_SIZE); - while (!buf.eof()) - { - const size_t capacity = max_buffered_size - out.size(); - if (capacity == 0) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", - max_buffered_size); - } - - const size_t to_read = std::min(tmp.size(), capacity); - const size_t n = buf.read(reinterpret_cast(tmp.data()), to_read); - if (n == 0) - break; - - out.insert(out.end(), tmp.data(), tmp.data() + n); - - /// If we filled the remaining capacity and the stream still has data, fail closed. - if (n == capacity && !buf.eof()) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", - max_buffered_size); - } - } -} - -std::vector readPuffinFooterBlobsFromSeekable(SeekableReadBuffer & seekable, size_t file_size) -{ - ScopedPuffinFileReadProfileEvent profile_event; - - if (file_size < 16) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); - - seekable.seek(0, SEEK_SET); - char magic_buf[4]; - seekable.readStrict(magic_buf, 4); - checkMagic(reinterpret_cast(magic_buf), "header"); - - seekable.seek(static_cast(file_size - PUFFIN_FOOTER_TRAILER_SIZE), SEEK_SET); - Int32 footer_length_signed = 0; - readBinaryLittleEndian(footer_length_signed, seekable); - - UInt8 flags[4] = {}; - seekable.readStrict(reinterpret_cast(flags), sizeof(flags)); - - char trailing_buf[4]; - seekable.readStrict(trailing_buf, 4); - checkMagic(reinterpret_cast(trailing_buf), "trailing"); - validatePuffinFooterFlags(flags); - - if (footer_length_signed <= 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); - - UInt64 footer_trailer_size = 0; - if (common::addOverflow( - static_cast(footer_length_signed), - static_cast(PUFFIN_FOOTER_TRAILER_SIZE), - footer_trailer_size) - || footer_trailer_size > file_size) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); - - const size_t footer_length = static_cast(footer_length_signed); - const size_t payload_start = file_size - PUFFIN_FOOTER_TRAILER_SIZE - footer_length; - /// Footer layout is Magic | FooterPayload | …; require a distinct footer-open Magic after the - /// file header Magic (payload_start == 4 would reuse the header Magic when blobs is empty). - if (payload_start < 2 * sizeof(PUFFIN_MAGIC)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin footer length: {}", footer_length_signed); - - if (footer_length > PUFFIN_FOOTER_MAX_PAYLOAD_SIZE) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin footer payload size {} exceeds absolute limit {}", - footer_length, - PUFFIN_FOOTER_MAX_PAYLOAD_SIZE); - - seekable.seek(static_cast(payload_start - sizeof(PUFFIN_MAGIC)), SEEK_SET); - char leading_magic_buf[4]; - seekable.readStrict(leading_magic_buf, 4); - checkMagic(reinterpret_cast(leading_magic_buf), "footer"); - - String footer_payload(footer_length, '\0'); - seekable.seek(static_cast(payload_start), SEEK_SET); - seekable.readStrict(footer_payload.data(), footer_length); - - String footer_json; - if ((flags[0] & PUFFIN_FOOTER_COMPRESSED_FLAG) != 0) - footer_json = decompressPuffinFooterPayload(footer_payload.data(), footer_payload.size()); - else - footer_json = std::move(footer_payload); - - const size_t blob_region_end = payload_start - sizeof(PUFFIN_MAGIC); - return parseFooterJSON(footer_json, blob_region_end); -} - -const PuffinBlob & bindDeletionVectorBlob( - const std::vector & blobs, - Int64 content_offset, - Int64 content_size_in_bytes, - std::string_view expected_referenced_data_file, - UInt64 expected_cardinality) -{ - const PuffinBlob * matched = nullptr; - size_t matched_index = 0; - - for (size_t i = 0; i < blobs.size(); ++i) - { - if (blobs[i].offset != content_offset || blobs[i].length != content_size_in_bytes) - continue; - - if (matched) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Multiple Puffin blobs claim offset {} length {}", - content_offset, - content_size_in_bytes); - } - - matched = &blobs[i]; - matched_index = i; - } - - if (!matched) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "No Puffin footer blob at offset {} length {}", - content_offset, - content_size_in_bytes); - } - - if (matched->type != "deletion-vector-v1") - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {} at offset {} length {} has type '{}', expected deletion-vector-v1", - matched_index, - content_offset, - content_size_in_bytes, - matched->type); - } - - if (!matched->compression_codec.empty()) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 must omit compression-codec", - matched_index); - } - - const auto ref_it = matched->properties.find("referenced-data-file"); - if (ref_it == matched->properties.end() || ref_it->second.empty()) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 missing required property 'referenced-data-file'", - matched_index); - } - - if (ref_it->second != expected_referenced_data_file) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {} referenced-data-file '{}' does not match expected data file '{}'", - matched_index, - ref_it->second, - expected_referenced_data_file); - } - - const auto card_it = matched->properties.find("cardinality"); - if (card_it == matched->properties.end() || card_it->second.empty()) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 missing required property 'cardinality'", - matched_index); - } - - UInt64 footer_cardinality = 0; - if (!tryParse(footer_cardinality, card_it->second)) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", - matched_index); - } - - if (footer_cardinality != expected_cardinality) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {} cardinality {} does not match expected cardinality {}", - matched_index, - footer_cardinality, - expected_cardinality); - } - - return *matched; -} - -} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFile.h b/src/Storages/ObjectStorage/DataLakes/PuffinFile.h deleted file mode 100644 index 8d340271a4d1..000000000000 --- a/src/Storages/ObjectStorage/DataLakes/PuffinFile.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -namespace DB -{ - -/// Leading / footer-open magic is always 4 bytes (`PFA1`). -constexpr size_t PUFFIN_MAGIC_SIZE = 4; -constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; -/// Absolute cap on footer payload size (uncompressed JSON bytes, or declared LZ4 contentSize). -constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; - -/// Non-seekable SQL path must buffer the whole file to reach the trailer. Cap total buffered size -/// so a crafted pipe cannot allocate unbounded memory before footer-length validation. -/// Sized for one max DV blob + max footer (header magic + blob + footer magic + payload + trailer). -constexpr size_t PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE = PUFFIN_MAGIC_SIZE + PUFFIN_DV_MAX_BLOB_SIZE - + PUFFIN_MAGIC_SIZE + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE + PUFFIN_FOOTER_TRAILER_SIZE; - -struct PuffinBlob -{ - String type; - Int64 snapshot_id = 0; - Int64 sequence_number = 0; - std::vector fields; - Int64 offset = 0; - Int64 length = 0; - String compression_codec; - std::map properties; -}; - -/// Append bytes from `buf` into `out` until EOF. Throws if `out` would exceed `max_buffered_size`. -void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size); - -/// Parse the Puffin footer of a seekable file and return blob metadata. -std::vector readPuffinFooterBlobsFromSeekable(SeekableReadBuffer & seekable, size_t file_size); - -/// Find the unique footer blob at (`content_offset`, `content_size_in_bytes`) and bind it as a -/// deletion-vector-v1 for `expected_referenced_data_file` with `expected_cardinality`. -/// Throws if the slice is missing, ambiguous, or does not match the expected DV identity. -const PuffinBlob & bindDeletionVectorBlob( - const std::vector & blobs, - Int64 content_offset, - Int64 content_size_in_bytes, - std::string_view expected_referenced_data_file, - UInt64 expected_cardinality); - -} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp index 43242b96bb36..80740ccf330b 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp @@ -3,7 +3,6 @@ #include #include #include -#include using namespace DB; diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp index d94e7304b31b..643613e6232f 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp @@ -1,8 +1,9 @@ #include +#include + #include #include -#include #include