diff --git a/CHANGELOG.md b/CHANGELOG.md index 54abc85..b50a934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ Notable changes are recorded from 0.5.0 onward. +## 0.7.0 + +### Changed + +- Minarrow upgraded to 0.18.1 and vec64 0.5.1. +- New `decimal` feature forwarding minarrow's `decimal` feature. Decimal32, Decimal64 and Decimal128 columns are supported in Arrow IPC and in Parquet, where they map to the DECIMAL logical type over INT32, INT64 and FIXED_LEN_BYTE_ARRAY. +- The Python package pins minarrow and minarrow-pyo3 at 0.18.1. +- Parquet writer improvements: + - The Parquet writer follows the Parquet value layout for nullable columns. + - Several improvements in categorical, time, and date roundtripping. + - Files with nulls written by earlier releases do not read back under this release. + - Replaced packing for Duration and Interval columns with `UnsupportedType` + +### Fixed + +- The Parquet reader failed with `UnexpectedEof` on files written by pyarrow. It now reads DataPageV1 pages with compressed levels, `RLE` booleans, dictionary-encoded columns of any physical type, and files with several row groups. + +## 0.6.1 + +### Changed + +- minarrow version bump to 0.17.0, vec64 0.5.0, arrow 59.2.0 and polars 0.55.2. + ## 0.6.0 ### Changed diff --git a/python/Cargo.lock b/python/Cargo.lock index f394dbb..c538bef 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -662,7 +662,7 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "lightstream" -version = "0.6.0" +version = "0.7.0" dependencies = [ "bytes", "fast-float2", @@ -692,7 +692,7 @@ dependencies = [ [[package]] name = "lightstream-py" -version = "0.6.0" +version = "0.7.0" dependencies = [ "futures-core", "lightstream", @@ -734,11 +734,10 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "minarrow" -version = "0.16.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec048e9b7a43abce573068b7f4e220b82a78e44d19c2888e31527efa7a4be109" +checksum = "e53f5031b7d16aa76bc8671171814419694c687f5160917a3668f010dab2b609" dependencies = [ - "libc", "log", "num-traits", "vec64", @@ -746,9 +745,9 @@ dependencies = [ [[package]] name = "minarrow-pyo3" -version = "0.16.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde6692dbbba1430e1205cb65c56420d3ff2440592ce982d98f2794a14a7b8ef" +checksum = "94e91845f51b0aab761625144ab1c71b989d5237d8cc6159245db07b4b78bc26" dependencies = [ "minarrow", "pyo3", @@ -1674,9 +1673,9 @@ dependencies = [ [[package]] name = "vec64" -version = "0.4.7" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14d6d15c0dbb3d2929a916fe20a15e7d04e787c9fbc29b3940e0654b11482c6" +checksum = "1aef6bbef159f21ac387220ebc71141524423f7886afd390bc7b140f12630425" [[package]] name = "version_check" diff --git a/python/Cargo.toml b/python/Cargo.toml index bf4ff3e..9aef560 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -2,7 +2,7 @@ cargo-features = ["trim-paths"] [package] name = "lightstream-py" -version = "0.6.0" +version = "0.7.0" edition = "2024" authors = ["Peter G. Bower"] license = "MPL-2.0" @@ -19,13 +19,13 @@ name = "lightstream_py" crate-type = ["cdylib", "rlib"] [dependencies] -lightstream = { version = "0.6", path = "../rust", features = ["csv", "datetime", "extended_categorical", "extended_numeric_types", "json", "mmap", "http", "parquet", "protocol", "quic", "snappy", "stdio", "tcp", "tls", "uds", "webtransport", "websocket", "zstd"] } +lightstream = { version = "0.7", path = "../rust", features = ["csv", "datetime", "decimal", "extended_categorical", "extended_numeric_types", "json", "mmap", "http", "parquet", "protocol", "quic", "snappy", "stdio", "tcp", "tls", "uds", "webtransport", "websocket", "zstd"] } # The categorical and numeric feature set mirrors the minarrow-py build. # minarrow-pyo3's dictionary-index conversion needs the extended features, # and they flow through lightstream's flags so its match arms gate in step # with minarrow's variants. -minarrow = { version = "0.16", features = ["chunked"] } -minarrow-pyo3 = { version = "0.16", features = ["extended_categorical", "extended_numeric_types"] } +minarrow = { version = "0.18.1", features = ["chunked"] } +minarrow-pyo3 = { version = "0.18.1", features = ["extended_categorical", "extended_numeric_types"] } futures-core = "0.3" pyo3 = { version = "0.29", features = ["abi3-py39"] } # The QUIC and WebTransport dependency pins mirror lightstream's, so diff --git a/python/pyproject.toml b/python/pyproject.toml index c2ae316..5c6f7a1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "lightstream-io" -version = "0.6.0" +version = "0.7.0" description = "Streaming Arrow I/O for Python - files, sockets, and network transports with zero-copy minarrow interop." readme = "README.md" requires-python = ">=3.9" @@ -20,7 +20,7 @@ classifiers = [ "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries", ] -dependencies = ["minarrow>=0.16"] +dependencies = ["minarrow>=0.18.1"] [project.urls] Homepage = "https://github.com/SpaceCell/lightstream" diff --git a/python/tests/test_files.py b/python/tests/test_files.py index 2f7be22..575bfd0 100644 --- a/python/tests/test_files.py +++ b/python/tests/test_files.py @@ -11,10 +11,12 @@ """ import gc +from decimal import Decimal import lightstream as ls import minarrow import pyarrow as pa +import pyarrow.parquet as pq import pytest @@ -224,6 +226,44 @@ def test_parquet_multi_write_consolidates(tmp_path): assert result.num_rows == 6 +def nullable_table(): + return pa.table( + { + "id": pa.array([1, None, 3, 4, None], type=pa.int64()), + "small": pa.array([None, -2, 3, -4, 5], type=pa.int32()), + "count": pa.array([1, 2, None, 4, 5], type=pa.uint32()), + "name": pa.array(["a", "b", None, "a", "c"], type=pa.string()), + "score": pa.array([1.5, None, 3.5, 4.5, 5.5], type=pa.float64()), + "ratio": pa.array([0.5, 1.5, 2.5, None, 4.5], type=pa.float32()), + "flag": pa.array([True, False, None, True, False], type=pa.bool_()), + "day": pa.array([1, None, 3, 4, 5], type=pa.date32()), + "at": pa.array([1_000, 2_000, None, 4_000, 5_000], type=pa.timestamp("ns")), + "amount": pa.array( + [Decimal("1.25"), None, Decimal("-3.50"), Decimal("0.01"), Decimal("99.99")], + type=pa.decimal128(10, 2), + ), + } + ) + + +def test_parquet_reads_pyarrow_file_with_nulls(tmp_path): + path = str(tmp_path / "pyarrow.parquet") + original = nullable_table() + pq.write_table(original, path) + + result = pa.table(ls.read(path).read_all()) + assert result.to_pydict() == original.to_pydict() + + +def test_parquet_written_with_nulls_reads_in_pyarrow(tmp_path): + path = str(tmp_path / "lightstream.parquet") + original = nullable_table() + with ls.write(path) as w: + w.write(original) + + assert pq.read_table(path).to_pydict() == original.to_pydict() + + @pytest.mark.parametrize("codec", ["zstd", "snappy"]) def test_parquet_compression(tmp_path, codec): path = str(tmp_path / f"quotes_{codec}.parquet") diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0d69e7c..9387c3a 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -48,9 +48,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -113,15 +113,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.104" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "ar_archive_writer" -version = "0.5.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" dependencies = [ "object", ] @@ -142,17 +142,23 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed51fe0f224d1d4ea768be38c51f9f831dee9d05c163c11fba0b8c44387b1fc3" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" -version = "0.7.8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ "arrow-arith", "arrow-array", @@ -169,9 +175,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -183,9 +189,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash", "arrow-buffer", @@ -202,9 +208,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", @@ -214,9 +220,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +241,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -248,9 +254,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0e6d452fff35cb4a3ef1a719d03fdfe653e590e8012b57d3057a316e295402" +checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" dependencies = [ "arrow-array", "arrow-buffer", @@ -269,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -283,9 +289,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ "arrow-array", "arrow-buffer", @@ -296,9 +302,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -309,18 +315,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", ] [[package]] name = "arrow-select" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash", "arrow-array", @@ -332,9 +338,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ "arrow-array", "arrow-buffer", @@ -349,9 +355,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -371,7 +377,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -383,7 +389,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -417,18 +423,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -458,9 +464,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" @@ -554,24 +560,25 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ "serde_core", ] [[package]] name = "blake3" -version = "1.8.7" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ + "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", ] [[package]] @@ -585,9 +592,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.12.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ "hybrid-array", ] @@ -600,35 +607,35 @@ checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e" [[package]] name = "bumpalo" -version = "3.20.3" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytemuck" -version = "1.25.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.12.0" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "bytes" -version = "1.12.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -650,9 +657,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -668,18 +675,18 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -734,18 +741,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstyle", "clap_lex", @@ -765,9 +772,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "compact_str" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -846,18 +853,18 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] @@ -900,18 +907,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.17" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.8" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -919,27 +926,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.21" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.14" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.23" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -959,18 +966,18 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" dependencies = [ "hybrid-array", ] [[package]] name = "data-encoding" -version = "2.11.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "debug_unsafe" @@ -978,37 +985,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror", -] - [[package]] name = "der-parser" version = "10.0.0" @@ -1018,7 +994,7 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint 0.4.8", + "num-bigint 0.4.6", "num-traits", "rusticata-macros", ] @@ -1028,6 +1004,9 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] [[package]] name = "digest" @@ -1045,9 +1024,9 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.1", + "block-buffer 0.12.0", "const-oid", - "crypto-common 0.2.2", + "crypto-common 0.2.1", ] [[package]] @@ -1061,13 +1040,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.7" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -1078,15 +1057,15 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.18.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "env_filter" -version = "2.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -1094,9 +1073,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.11" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -1129,10 +1108,11 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.2" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" dependencies = [ + "concurrent-queue", "parking", "pin-project-lite", ] @@ -1155,21 +1135,21 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fast-float2" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastrand" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flatbuffers" @@ -1177,19 +1157,18 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", - "zlib-rs", ] [[package]] @@ -1240,9 +1219,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1255,9 +1234,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1265,15 +1244,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1282,38 +1261,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "futures-sink" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1356,36 +1335,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasip2", + "wasip3", "wasm-bindgen", ] [[package]] name = "glob" -version = "0.3.4" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.19" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1489,9 +1472,9 @@ checksum = "1a9fcbcc408c5526c3ab80d534e5c86e7967c1fb7aa0a8c76abd1edc27deb877" [[package]] name = "http" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", "itoa", @@ -1499,9 +1482,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", @@ -1509,9 +1492,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.5" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", @@ -1534,24 +1517,24 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1615,7 +1598,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.5", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -1647,9 +1630,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", @@ -1661,9 +1644,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1674,9 +1657,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1688,17 +1671,16 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1709,15 +1691,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1728,6 +1710,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" version = "1.1.0" @@ -1751,9 +1739,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.2" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -1773,9 +1761,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.2" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" @@ -1809,12 +1797,10 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ - "defmt", - "jiff-core", "jiff-static", "log", "portable-atomic", @@ -1822,45 +1808,36 @@ dependencies = [ "serde_core", ] -[[package]] -name = "jiff-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" -dependencies = [ - "defmt", -] - [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ - "jiff-core", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "jobserver" -version = "0.1.35" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", + "once_cell", "wasm-bindgen", ] @@ -1870,6 +1847,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lexical-core" version = "1.0.6" @@ -1929,9 +1912,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -1941,7 +1924,7 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "lightstream" -version = "0.6.1" +version = "0.7.0" dependencies = [ "arrow", "arrow-flight", @@ -1992,9 +1975,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2007,9 +1990,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru-slab" @@ -2050,9 +2033,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" -version = "0.9.11" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", ] @@ -2065,9 +2048,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minarrow" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a92f9ea49a9b48aa50d62d75824dd14c5802229c6f18f10d6cd880f390d637" +checksum = "e53f5031b7d16aa76bc8671171814419694c687f5160917a3668f010dab2b609" dependencies = [ "arrow", "arrow-schema", @@ -2083,9 +2066,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.9.1" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "simd-adler32", @@ -2093,9 +2076,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -2132,9 +2115,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.8" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -2161,9 +2144,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-derive" @@ -2173,14 +2156,14 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "num-integer" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ "num-traits", ] @@ -2201,7 +2184,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", ] [[package]] @@ -2216,9 +2199,9 @@ dependencies = [ [[package]] name = "object" -version = "0.39.1" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] @@ -2245,7 +2228,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.2", + "rand 0.10.1", "reqwest", "ring", "serde", @@ -2262,9 +2245,9 @@ dependencies = [ [[package]] name = "octets" -version = "0.3.7" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d59d30d3ad7f7e1c9a66e51c9cb71ae1c794b28ffe7053fd785defb0d180069" +checksum = "8311fa8ab7a57759b4ff1f851a3048d9ef0effaa0130726426b742d26d8a88e7" [[package]] name = "oid-registry" @@ -2348,16 +2331,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "pem" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" -dependencies = [ - "base64 0.23.1", - "serde_core", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2399,7 +2372,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2410,9 +2383,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "planus" @@ -2459,7 +2432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52d3ed4e6b3917427f6d3c43edbd2740babe228bb4ccfa3431eac105844045d" dependencies = [ "getrandom 0.2.17", - "getrandom 0.4.3", + "getrandom 0.4.2", "polars-arrow", "polars-buffer", "polars-compute", @@ -2480,7 +2453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5ab55460ca2553d1a2bbaf2046998cfbbb685bc18d1e168b4367b63301eb271" dependencies = [ "atoi_simd", - "bitflags 2.13.1", + "bitflags 2.11.1", "bytemuck", "bytes", "chrono", @@ -2489,7 +2462,7 @@ dependencies = [ "either", "ethnum", "getrandom 0.2.17", - "getrandom 0.4.3", + "getrandom 0.4.2", "half", "hashbrown 0.17.1", "itoa", @@ -2533,7 +2506,7 @@ dependencies = [ "polars-config", "polars-error", "polars-utils", - "rand 0.10.2", + "rand 0.10.1", "slotmap", "tokio", ] @@ -2569,7 +2542,7 @@ dependencies = [ "polars-buffer", "polars-error", "polars-utils", - "rand 0.10.2", + "rand 0.10.1", "serde", "strength_reduce", "strum_macros", @@ -2593,13 +2566,13 @@ version = "0.55.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac24a25f4c2696089bb9059e4b45338ea74833a79b9552fef066d6f2cfb3aaed" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "boxcar", "bytemuck", "chrono", "chrono-tz", "either", - "getrandom 0.4.3", + "getrandom 0.4.2", "hashbrown 0.17.1", "indexmap", "itoa", @@ -2614,7 +2587,7 @@ dependencies = [ "polars-row", "polars-schema", "polars-utils", - "rand 0.10.2", + "rand 0.10.1", "rand_distr", "rayon", "regex", @@ -2662,7 +2635,7 @@ version = "0.55.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b59262210ac41f3b30f49f46855501206eba27baaa6bd919545703bc706dd617" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "hashbrown 0.17.1", "num-traits", "polars-arrow", @@ -2675,7 +2648,7 @@ dependencies = [ "polars-row", "polars-time", "polars-utils", - "rand 0.10.2", + "rand 0.10.1", "rayon", "recursive", "regex", @@ -2717,7 +2690,7 @@ dependencies = [ "polars-parquet", "polars-schema", "polars-utils", - "rand 0.10.2", + "rand 0.10.1", "rayon", "regex", "reqwest", @@ -2734,7 +2707,7 @@ version = "0.55.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79654a98520b5df9be08ce5004be90a5c8004d0f73d54500c1aa923874ec84f3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "chrono", "either", "memchr", @@ -2790,7 +2763,7 @@ dependencies = [ "polars-error", "polars-io", "polars-utils", - "rand 0.10.2", + "rand 0.10.1", "rand_distr", "thread_local", "tokio", @@ -2873,7 +2846,7 @@ version = "0.55.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "467bd857c46405807e6645942be2de284aa43bc503fc6d20efb16e1adf25b7a6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "blake3", "bytemuck", "bytes", @@ -2914,7 +2887,7 @@ version = "0.55.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71ac05187ac33abcfe9b7c997f0cd2ea9bfe8e04e9018d343e16fb2b64e2f5bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "bytemuck", "polars-arrow", "polars-buffer", @@ -2943,7 +2916,7 @@ version = "0.55.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae4b287917a1c50b9fab9618c32d3ac8c4b18ef1c61aded55d065c97c921c793" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "hex", "polars-core", "polars-error", @@ -2965,7 +2938,7 @@ checksum = "b173c0c52f53ca930f3b59bc56a5854a650eb9555d67bba9022051714bc0b88f" dependencies = [ "async-channel", "async-trait", - "bitflags 2.13.1", + "bitflags 2.11.1", "bytes", "chrono", "chrono-tz", @@ -3048,7 +3021,7 @@ dependencies = [ "num-traits", "polars-config", "polars-error", - "rand 0.10.2", + "rand 0.10.1", "raw-cpuid", "rayon", "regex", @@ -3066,24 +3039,24 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.15.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.8" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -3103,20 +3076,30 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" -version = "1.0.107" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.14.4" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -3124,31 +3107,31 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.4" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "prost-types" -version = "0.14.4" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] [[package]] name = "psm" -version = "0.1.32" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" dependencies = [ "ar_archive_writer", "cc", @@ -3166,9 +3149,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -3177,7 +3160,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.5", + "socket2 0.6.3", "thiserror", "tokio", "tracing", @@ -3186,15 +3169,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", - "getrandom 0.4.3", + "getrandom 0.3.4", "lru-slab", - "rand 0.10.2", - "rand_pcg", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -3208,23 +3190,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.5", + "socket2 0.6.3", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.47" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3243,9 +3225,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -3253,12 +3235,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", - "getrandom 0.4.3", + "getrandom 0.4.2", "rand_core 0.10.1", ] @@ -3294,16 +3276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" dependencies = [ "num-traits", - "rand 0.10.2", -] - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", + "rand 0.10.1", ] [[package]] @@ -3312,7 +3285,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", ] [[package]] @@ -3337,11 +3310,11 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.10" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ - "pem 4.0.0", + "pem", "ring", "rustls-pki-types", "time", @@ -3366,7 +3339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3375,34 +3348,34 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", ] [[package]] name = "ref-cast" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "regex" -version = "1.13.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -3412,9 +3385,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3423,9 +3396,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.11" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" @@ -3504,9 +3477,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.3" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -3532,7 +3505,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -3541,9 +3514,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.44" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "once_cell", "ring", @@ -3555,9 +3528,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3567,9 +3540,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -3577,9 +3550,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -3588,9 +3561,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.23" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3628,7 +3601,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "core-foundation", "core-foundation-sys", "libc", @@ -3653,9 +3626,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -3663,29 +3636,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", @@ -3719,9 +3692,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.7" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -3735,15 +3708,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] [[package]] name = "shlex" -version = "2.0.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" @@ -3767,18 +3740,19 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.10" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simd-json" -version = "0.17.3" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32d7ab2678282d21e53374fbead7119b7eacbede73685dcaac472870a29a11c" +checksum = "4255126f310d2ba20048db6321c81ab376f6a6735608bf11f0785c41f01f64e3" dependencies = [ "ahash", "halfbrown", + "once_cell", "ref-cast", "serde", "serde_json", @@ -3815,15 +3789,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "snap" -version = "1.1.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" @@ -3837,9 +3811,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3864,7 +3838,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3875,9 +3849,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.25" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" dependencies = [ "cc", "cfg-if", @@ -3922,7 +3896,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3933,20 +3907,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.5" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -3970,7 +3933,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3994,7 +3957,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4002,22 +3965,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -4031,11 +3994,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.55" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde_core", @@ -4045,15 +4009,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.9" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.32" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -4070,9 +4034,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -4090,9 +4054,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4105,35 +4069,35 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", - "socket2 0.6.5", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -4141,9 +4105,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.19" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -4182,14 +4146,13 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", "futures-sink", - "libc", "pin-project-lite", "tokio", ] @@ -4213,7 +4176,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.5", + "socket2 0.6.3", "sync_wrapper", "tokio", "tokio-stream", @@ -4255,11 +4218,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.11" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -4302,7 +4265,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4331,7 +4294,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.5", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1", @@ -4341,9 +4304,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.20.1" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicode-ident" @@ -4371,9 +4334,15 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.13.3" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -4419,11 +4388,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.4.2", "js-sys", "serde_core", "wasm-bindgen", @@ -4431,9 +4400,9 @@ dependencies = [ [[package]] name = "value-trait" -version = "0.12.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f4b4a98dfe54bc9ed3641af7ffcb837240269627dbd5cb047d13daa39736cc" +checksum = "8e80f0c733af0720a501b3905d22e2f97662d8eacfe082a75ed7ffb5ab08cb59" dependencies = [ "float-cmp", "halfbrown", @@ -4489,18 +4458,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.4+wasi-0.2.12" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -4511,9 +4489,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -4521,9 +4499,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4531,26 +4509,48 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -4564,11 +4564,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -4590,14 +4602,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.7", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -4686,7 +4698,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4697,7 +4709,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4740,7 +4752,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -4758,14 +4779,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -4783,75 +4821,211 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wtransport" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4273ce3157a3262a68665f8d3f20a0ac0c5b8a69ffd67f05ae986832ebec036" +checksum = "ea4aacf790813ee1956751491800537f4e04af7557b7b370501ccbfbc85963e4" dependencies = [ "bytes", - "pem 3.0.6", + "pem", "quinn", "rcgen", "rustls", "rustls-native-certs", "rustls-pki-types", "sha2", - "socket2 0.6.5", + "socket2 0.6.3", "thiserror", "time", "tokio", @@ -4863,9 +5037,9 @@ dependencies = [ [[package]] name = "wtransport-proto" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aad9059572c7dbd6901ccef37f3b7321678cd708dcf58a64b1921dbeab7bfede" +checksum = "d5867c629e4252f7439d82315923daaf27f4fa442410d51b78ab93ef4c432a11" dependencies = [ "httlib-huffman", "octets", @@ -4893,9 +5067,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yasna" @@ -4909,9 +5083,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4926,35 +5100,35 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "zerofrom" -version = "0.1.8" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] @@ -4967,21 +5141,21 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -4990,9 +5164,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -5001,26 +5175,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - [[package]] name = "zmij" -version = "1.0.23" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" @@ -5033,18 +5201,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.3.0" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f6ce1da..7bf99ed 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightstream" -version = "0.6.1" +version = "0.7.0" edition = "2024" license = "MPL-2.0" keywords = [ @@ -42,8 +42,8 @@ flatbuffers = "25.12.19" libc = { version = "0.2.183", optional = true } futures-sink = "0.3.32" log = "0.4" -minarrow = { version = "0.18.0", features = ["chunked", "views", "select", "size"], default-features = false } -vec64 = { version = "0.5.0" } +minarrow = { version = "0.18.1", features = ["chunked", "views", "select", "size"], default-features = false } +vec64 = { version = "0.5.1" } snap = { version = "1.0", optional = true } tokio-tungstenite = { version = "0.28", optional = true } zstd = { version = "0.13", optional = true } @@ -149,8 +149,8 @@ default_categorical_8 = ["minarrow/default_categorical_8"] extended_categorical = ["default_categorical_8", "minarrow/extended_categorical"] large_string = ["minarrow/large_string"] datetime = ["minarrow/datetime"] -extended_numeric_types = ["minarrow/extended_numeric_types"] decimal = ["minarrow/decimal"] +extended_numeric_types = ["minarrow/extended_numeric_types"] lbuffer = ["minarrow/lbuffer"] # Schema-level Table metadata as key-value pairs table_metadata = ["minarrow/table_metadata"] diff --git a/rust/README.md b/rust/README.md index 42c613e..98e9d64 100644 --- a/rust/README.md +++ b/rust/README.md @@ -200,7 +200,7 @@ cargo run --example uds_arrow --features uds | `msgpack`, `protobuf` | Message encodings (enables `protocol`) | | `tls` | TLS for TCP, WebSocket, HTTP/2 | | `io_uring` | Linux async I/O (experimental) | -| `datetime`, `large_string`, `extended_numeric_types` | Schema extensions | +| `datetime`, `decimal`, `large_string`, `extended_numeric_types` | Schema extensions | ## Buffering and safety diff --git a/rust/pyarrow-roundtrip/generate_pyarrow_parquet_files.py b/rust/pyarrow-roundtrip/generate_pyarrow_parquet_files.py new file mode 100644 index 0000000..e30de12 --- /dev/null +++ b/rust/pyarrow-roundtrip/generate_pyarrow_parquet_files.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Generate the PyArrow Parquet fixture files for tests/pyarrow_parquet.rs. + +Each file exercises one combination of the page layouts pyarrow can write, +so the reader is checked against real parquet-cpp output rather than +lightstream's own encoder: + + - pyarrow_simple.parquet: three required columns written with pyarrow's + defaults (Snappy, dictionary encoding, DataPageV1) + - pyarrow_nullable_row_groups.parquet: nullable columns with nulls, split + across three row groups, written with pyarrow's defaults + - pyarrow_plain_v2.parquet: the same nullable data with dictionary + encoding off, DataPageV2 and no compression + - pyarrow_dictionary_v2.parquet: the same nullable data with dictionary + encoding on, DataPageV2 and Snappy + - pyarrow_temporal_decimal.parquet: date, time, timestamp and decimal + columns with nulls, written with pyarrow's defaults. Nanosecond units + and decimals exercise the LogicalType annotation + +The fixtures are committed alongside this script. Run from rust/ to +regenerate them: + + python3 pyarrow-roundtrip/generate_pyarrow_parquet_files.py +""" + +import os +from decimal import Decimal + +import pyarrow as pa +import pyarrow.parquet as pq + +OUT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def simple_table() -> pa.Table: + """The three-column table from the original defect report.""" + return pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()), + "name": pa.array(["Alice", "Bob", "Charlie", "Diana", "Eve"], type=pa.string()), + "score": pa.array([85.5, 92.0, 78.3, 95.1, 88.7], type=pa.float64()), + } + ) + + +def nullable_table() -> pa.Table: + """Seven rows over the basic types, matching expected_nullable_table in Rust.""" + return pa.table( + { + "int32": pa.array([1, None, 3, 4, None, 6, 7], type=pa.int32()), + "int64": pa.array([100, 101, None, 103, 104, 105, None], type=pa.int64()), + "float32": pa.array([0.5, 1.5, 2.5, None, 4.5, 5.5, 6.5], type=pa.float32()), + "float64": pa.array([None, -1.0, -2.0, -3.0, -4.0, None, -6.0], type=pa.float64()), + "bool": pa.array([True, False, None, True, False, True, None], type=pa.bool_()), + "string": pa.array(["a", "b", "a", None, "c", "b", "a"], type=pa.string()), + } + ) + + +def temporal_decimal_table() -> pa.Table: + """Five rows of temporal and decimal types, matching the expectations in + reads_pyarrow_temporal_and_decimal_columns in Rust.""" + return pa.table( + { + "date": pa.array([0, 1, None, 19_000, -5], type=pa.date32()), + "time_ms": pa.array([0, 1_000, None, 43_200_000, 86_399_999], type=pa.time32("ms")), + "time_us": pa.array([0, None, 2_000_000, 43_200_000_000, 86_399_999_999], type=pa.time64("us")), + "ts_ms": pa.array([0, 1_700_000_000_000, None, -1, 86_400_000], type=pa.timestamp("ms")), + "ts_us": pa.array([0, 1_700_000_000_000_000, None, -1, 1], type=pa.timestamp("us")), + "ts_ns": pa.array([0, 1_700_000_000_000_000_000, None, -1, 1], type=pa.timestamp("ns")), + "ts_us_utc": pa.array([0, 1_700_000_000_000_000, None, -1, 1], type=pa.timestamp("us", tz="UTC")), + "dec32": pa.array( + [Decimal("1.25"), None, Decimal("-3.50"), Decimal("0.01"), Decimal("99999.99")], + type=pa.decimal32(7, 2), + ), + "dec64": pa.array( + [Decimal("1.2345"), Decimal("-1.0000"), None, Decimal("0.0001"), Decimal("12345678901234.5678")], + type=pa.decimal64(18, 4), + ), + "dec128": pa.array( + [Decimal("1.000001"), None, Decimal("-1.000001"), Decimal("0.000000"), Decimal("12345678901234567890123456789012.123456")], + type=pa.decimal128(38, 6), + ), + } + ) + + +def main() -> None: + simple = simple_table() + pq.write_table(simple, os.path.join(OUT_DIR, "pyarrow_simple.parquet")) + + nullable = nullable_table() + pq.write_table( + nullable, + os.path.join(OUT_DIR, "pyarrow_nullable_row_groups.parquet"), + row_group_size=3, + ) + pq.write_table( + nullable, + os.path.join(OUT_DIR, "pyarrow_plain_v2.parquet"), + use_dictionary=False, + data_page_version="2.0", + compression="none", + ) + pq.write_table( + nullable, + os.path.join(OUT_DIR, "pyarrow_dictionary_v2.parquet"), + data_page_version="2.0", + compression="snappy", + ) + pq.write_table( + temporal_decimal_table(), + os.path.join(OUT_DIR, "pyarrow_temporal_decimal.parquet"), + ) + for name in sorted(os.listdir(OUT_DIR)): + if name.startswith("pyarrow_") and name.endswith(".parquet"): + print(f"wrote {name}") + + +if __name__ == "__main__": + main() diff --git a/rust/pyarrow-roundtrip/pyarrow_dictionary_v2.parquet b/rust/pyarrow-roundtrip/pyarrow_dictionary_v2.parquet new file mode 100644 index 0000000..0f2fe50 Binary files /dev/null and b/rust/pyarrow-roundtrip/pyarrow_dictionary_v2.parquet differ diff --git a/rust/pyarrow-roundtrip/pyarrow_nullable_row_groups.parquet b/rust/pyarrow-roundtrip/pyarrow_nullable_row_groups.parquet new file mode 100644 index 0000000..472c26d Binary files /dev/null and b/rust/pyarrow-roundtrip/pyarrow_nullable_row_groups.parquet differ diff --git a/rust/pyarrow-roundtrip/pyarrow_plain_v2.parquet b/rust/pyarrow-roundtrip/pyarrow_plain_v2.parquet new file mode 100644 index 0000000..cc6d28b Binary files /dev/null and b/rust/pyarrow-roundtrip/pyarrow_plain_v2.parquet differ diff --git a/rust/pyarrow-roundtrip/pyarrow_simple.parquet b/rust/pyarrow-roundtrip/pyarrow_simple.parquet new file mode 100644 index 0000000..438815d Binary files /dev/null and b/rust/pyarrow-roundtrip/pyarrow_simple.parquet differ diff --git a/rust/pyarrow-roundtrip/pyarrow_temporal_decimal.parquet b/rust/pyarrow-roundtrip/pyarrow_temporal_decimal.parquet new file mode 100644 index 0000000..86e755c Binary files /dev/null and b/rust/pyarrow-roundtrip/pyarrow_temporal_decimal.parquet differ diff --git a/rust/src/constants.rs b/rust/src/constants.rs index e44ba5f..ba2dcc2 100644 --- a/rust/src/constants.rs +++ b/rust/src/constants.rs @@ -142,3 +142,6 @@ pub const METADATA_SIZE_PREFIX: usize = 4; /// Required “PAR1” marker written at Parquet file head and tail. pub const PARQUET_MAGIC: &[u8; 4] = b"PAR1"; + +/// Writer identity recorded in the `created_by` field of the Parquet footer. +pub const PARQUET_CREATED_BY: &str = "parquet_writer-v2"; diff --git a/rust/src/models/encoders/parquet/data.rs b/rust/src/models/encoders/parquet/data.rs index 00268cb..5098e1d 100644 --- a/rust/src/models/encoders/parquet/data.rs +++ b/rust/src/models/encoders/parquet/data.rs @@ -18,6 +18,7 @@ use minarrow::Bitmask; use crate::error::IoError; +use crate::models::types::parquet::ParquetPhysicalType; /// Encode `i32` values using Parquet plain little-endian format, appending to `out`. pub fn encode_int32_plain(data: &[i32], out: &mut Vec) { @@ -81,20 +82,23 @@ pub fn encode_float64_plain(data: &[f64], out: &mut Vec) { // // We only support RLE encoding for Categorical types at the present time. -/// Encode a boolean column bit-packed (LSB-first), respecting `null_mask`, appending to `out`. +/// Encode a boolean column bit-packed (LSB-first), appending to `out`. +/// +/// Null slots are omitted, so the output holds one bit per non-null value +/// as the Parquet value section requires. pub fn encode_bool_bitpacked( values: &Bitmask, null_mask: Option<&Bitmask>, len: usize, out: &mut Vec, ) { - //out.clear(); let mut byte = 0u8; let mut bit = 0; for i in 0..len { - let valid = null_mask.is_none_or(|m| m.get(i)); - let v = if valid { values.get(i) } else { false }; - if v { + if !null_mask.is_none_or(|m| m.get(i)) { + continue; + } + if values.get(i) { byte |= 1 << bit; } bit += 1; @@ -111,9 +115,10 @@ pub fn encode_bool_bitpacked( // UTF-8 strings -/// Encode String32 (UTF-8) using length-prefix (u32 LE) per row +/// Encode String32 (UTF-8) using length-prefix (u32 LE) per value /// -/// Nulls emit zero length. +/// Null slots are omitted, so the output holds one entry per non-null +/// value as the Parquet value section requires. pub fn encode_string_plain( offsets: &[u32], values: &[u8], @@ -122,23 +127,21 @@ pub fn encode_string_plain( out: &mut Vec, ) -> Result<(), IoError> { for i in 0..len { - // always emit a 4-byte length prefix - let valid = null_mask.is_none_or(|m| m.get(i)); + if !null_mask.is_none_or(|m| m.get(i)) { + continue; + } let start = offsets[i] as usize; let end = offsets[i + 1] as usize; - let s_len = if valid { end - start } else { 0 }; - out.extend_from_slice(&(s_len as u32).to_le_bytes()); - // only write the bytes for non-null - if valid { - out.extend_from_slice(&values[start..end]); - } + out.extend_from_slice(&((end - start) as u32).to_le_bytes()); + out.extend_from_slice(&values[start..end]); } Ok(()) } /// Encode LargeString (i.e., UTF-8, 64-bit offsets) as length-prefix (u32 LE) /// -/// Nulls emit zero length. +/// Null slots are omitted, so the output holds one entry per non-null +/// value as the Parquet value section requires. #[cfg(feature = "large_string")] pub fn encode_large_string_plain( offsets: &[u64], @@ -148,21 +151,63 @@ pub fn encode_large_string_plain( out: &mut Vec, ) -> Result<(), IoError> { for i in 0..len { - let valid = null_mask.is_none_or(|m| m.get(i)); + if !null_mask.is_none_or(|m| m.get(i)) { + continue; + } let start = offsets[i] as usize; let end = offsets[i + 1] as usize; - let s_len = if valid { end - start } else { 0 }; - if valid && s_len > u32::MAX as usize { + let s_len = end - start; + if s_len > u32::MAX as usize { return Err(IoError::InputDataError(format!( "string >4 GiB ({} bytes)", s_len ))); } - // length prefix for every row out.extend_from_slice(&(s_len as u32).to_le_bytes()); - // actual bytes only if non-null - if valid { - out.extend_from_slice(&values[start..end]); + out.extend_from_slice(&values[start..end]); + } + Ok(()) +} + +// Temporal values + +/// Plain-encode temporal values in the unit of the column's Parquet +/// annotation. +/// +/// `mul` and `div` come from `temporal_unit_scale` and carry each value +/// from its Arrow unit into the Parquet unit. `physical` selects the +/// storage width: INT32 for DATE and TIME(MILLIS), INT64 for the other +/// units. A value outside the INT32 range is an `InputDataError`. +pub(crate) fn encode_temporal_plain>( + data: &[T], + mul: i64, + div: i64, + physical: ParquetPhysicalType, + out: &mut Vec, +) -> Result<(), IoError> { + match physical { + ParquetPhysicalType::Int32 => { + out.reserve(data.len() * 4); + for &v in data { + let scaled = v.into() * mul / div; + let narrowed = i32::try_from(scaled).map_err(|_| { + IoError::InputDataError(format!( + "temporal value {scaled} does not fit the INT32 storage of its Parquet type" + )) + })?; + out.extend_from_slice(&narrowed.to_le_bytes()); + } + } + ParquetPhysicalType::Int64 => { + out.reserve(data.len() * 8); + for &v in data { + out.extend_from_slice(&(v.into() * mul / div).to_le_bytes()); + } + } + other => { + return Err(IoError::Internal(format!( + "temporal column mapped to physical type {other:?}" + ))); } } Ok(()) @@ -388,8 +433,8 @@ mod tests { #[test] fn test_encode_bool_bitpacked_with_nulls() { - // same values but pretend positions 1 and 3 are null ⇒ should be written as false - let values = vec64![true, true, true, false]; + // positions 1 and 3 are null, so only positions 0, 2 and 4 are packed + let values = vec64![true, true, false, false, true]; let mut nulls = Bitmask::new_set_all(values.len(), true); nulls.set_false(1); nulls.set_false(3); @@ -401,12 +446,32 @@ mod tests { &mut buf, ); - // check that bits at 1 and 3 are 0: - let byte = buf[0]; - assert_eq!((byte >> 0) & 1, 1); // idx0 valid, true - assert_eq!((byte >> 1) & 1, 0); // idx1 null -> treated as false - assert_eq!((byte >> 2) & 1, 1); // idx2 - assert_eq!((byte >> 3) & 1, 0); // idx3 null + // three packed bits: idx0 true, idx2 false, idx4 true + assert_eq!(buf, vec![0b101]); + } + + #[test] + fn test_encode_string_plain_omits_nulls() { + let slices = ["foo", "", "rust"]; + let mut offsets = Vec::with_capacity(slices.len() + 1); + offsets.push(0); + let mut values = Vec::new(); + for s in &slices { + values.extend_from_slice(s.as_bytes()); + offsets.push(values.len() as u32); + } + let mut nulls = Bitmask::new_set_all(slices.len(), true); + nulls.set_false(1); + let mut buf = Vec::new(); + encode_string_plain(&offsets, &values, Some(&nulls), slices.len(), &mut buf).unwrap(); + + // two entries: "foo" and "rust", the null slot has no length prefix + let mut expected = Vec::new(); + expected.extend_from_slice(&3u32.to_le_bytes()); + expected.extend_from_slice(b"foo"); + expected.extend_from_slice(&4u32.to_le_bytes()); + expected.extend_from_slice(b"rust"); + assert_eq!(buf, expected); } #[test] diff --git a/rust/src/models/encoders/parquet/metadata.rs b/rust/src/models/encoders/parquet/metadata.rs index e2bb9ae..d803e14 100644 --- a/rust/src/models/encoders/parquet/metadata.rs +++ b/rust/src/models/encoders/parquet/metadata.rs @@ -18,7 +18,7 @@ use std::io::{Seek, Write}; use crate::constants::PARQUET_MAGIC; use crate::error::IoError; -use crate::models::types::parquet::{ParquetEncoding, ParquetPhysicalType}; +use crate::models::types::parquet::{ParquetEncoding, ParquetLogicalType, ParquetPhysicalType}; // --------------------- Structs ------------------------------------ // @@ -50,6 +50,10 @@ pub(crate) struct SchemaElement { pub type_: Option, /// Legacy converted type ID (if any). pub converted_type: Option, + /// `LogicalType` annotation (field 10). Written alongside the converted + /// type so readers of either generation see the annotation. Nanosecond + /// units exist only here. + pub logical_type: Option, /// Type length (e.g. for FIXED_LEN_BYTE_ARRAY). pub type_length: Option, /// Decimal precision (if applicable). @@ -106,8 +110,6 @@ pub(crate) struct ColumnMetadata { pub dictionary_page_offset: Option, /// Optional per-column statistics. pub statistics: Option, - /// Definition level (REQUIRED/OPTIONAL/REPEATED encoded level). - pub definition_level: i32, } /// Parquet statistics for a column (min/max, null/unique counts). @@ -438,11 +440,94 @@ impl SchemaElement { thrift_write_field_i32(&mut w, &mut last, 9, id); } + // [10] LogicalType - a union struct whose single set field selects + // the annotation. Empty structs mark STRING and DATE. TIME and + // TIMESTAMP carry isAdjustedToUTC and a TimeUnit union. INTEGER + // carries the bit width and sign. DECIMAL carries scale then + // precision. + if let Some(logical) = &self.logical_type + && let Some(union_id) = logical_type_union_id(logical) + { + thrift_write_field_struct_begin(&mut w, &mut last, 10); + let mut union_last = 0i16; + thrift_write_field_struct_begin(&mut w, &mut union_last, union_id); + let mut inner = 0i16; + match logical { + #[cfg(feature = "datetime")] + ParquetLogicalType::TimestampMillis { utc } + | ParquetLogicalType::TimestampMicros { utc } + | ParquetLogicalType::TimestampNanos { utc } => { + thrift_write_field_bool(&mut w, &mut inner, 1, *utc); + thrift_write_field_struct_begin(&mut w, &mut inner, 2); + let mut unit_last = 0i16; + thrift_write_field_struct_begin(&mut w, &mut unit_last, time_unit_union_id(logical)); + thrift_write_field_stop(&mut w); + thrift_write_field_stop(&mut w); + } + // Time of day has no zone, so isAdjustedToUTC is false. + #[cfg(feature = "datetime")] + ParquetLogicalType::TimeMillis + | ParquetLogicalType::TimeMicros + | ParquetLogicalType::TimeNanos => { + thrift_write_field_bool(&mut w, &mut inner, 1, false); + thrift_write_field_struct_begin(&mut w, &mut inner, 2); + let mut unit_last = 0i16; + thrift_write_field_struct_begin(&mut w, &mut unit_last, time_unit_union_id(logical)); + thrift_write_field_stop(&mut w); + thrift_write_field_stop(&mut w); + } + ParquetLogicalType::IntType { bit_width, is_signed } => { + thrift_write_field_i8(&mut w, &mut inner, 1, *bit_width as i8); + thrift_write_field_bool(&mut w, &mut inner, 2, *is_signed); + } + #[cfg(feature = "decimal")] + ParquetLogicalType::Decimal { precision, scale } => { + thrift_write_field_i32(&mut w, &mut inner, 1, *scale as i32); + thrift_write_field_i32(&mut w, &mut inner, 2, *precision as i32); + } + _ => {} + } + thrift_write_field_stop(&mut w); + thrift_write_field_stop(&mut w); + } + thrift_write_field_stop(&mut w); Ok(()) } } +/// Field id of the `LogicalType` union member for a logical annotation. +/// `None` for annotations with no `LogicalType` form. +fn logical_type_union_id(logical: &ParquetLogicalType) -> Option { + Some(match logical { + ParquetLogicalType::NoneType => return None, + ParquetLogicalType::Utf8 => 1, + #[cfg(feature = "decimal")] + ParquetLogicalType::Decimal { .. } => 5, + #[cfg(feature = "datetime")] + ParquetLogicalType::Date32 => 6, + #[cfg(feature = "datetime")] + ParquetLogicalType::TimeMillis + | ParquetLogicalType::TimeMicros + | ParquetLogicalType::TimeNanos => 7, + #[cfg(feature = "datetime")] + ParquetLogicalType::TimestampMillis { .. } + | ParquetLogicalType::TimestampMicros { .. } + | ParquetLogicalType::TimestampNanos { .. } => 8, + ParquetLogicalType::IntType { .. } => 10, + }) +} + +/// Field id of the `TimeUnit` union member: MILLIS 1, MICROS 2, NANOS 3. +#[cfg(feature = "datetime")] +fn time_unit_union_id(logical: &ParquetLogicalType) -> i16 { + match logical { + ParquetLogicalType::TimestampMillis { .. } | ParquetLogicalType::TimeMillis => 1, + ParquetLogicalType::TimestampMicros { .. } | ParquetLogicalType::TimeMicros => 2, + _ => 3, + } +} + impl RowGroupMeta { /// Write a row group descriptor via TCompactProtocol. pub fn write(&self, mut w: W) -> Result<(), IoError> { @@ -575,6 +660,7 @@ impl Statistics { /// Compact-protocol element and field type identifiers from the Thrift spec. const TC_BOOL_TRUE: u8 = 1; const TC_BOOL_FALSE: u8 = 2; +const TC_BYTE: u8 = 3; const TC_I32: u8 = 5; const TC_I64: u8 = 6; const TC_BINARY: u8 = 8; @@ -637,6 +723,11 @@ fn thrift_write_list_header(w: &mut W, elem_type: u8, len: usize) { fn thrift_write_field_stop(w: &mut W) { w.write_all(&[0]).unwrap(); } +/// Write a byte field as one raw byte. +fn thrift_write_field_i8(w: &mut W, last: &mut i16, id: i16, v: i8) { + thrift_write_field_header(w, last, id, TC_BYTE); + w.write_all(&[v as u8]).unwrap(); +} /// Write an i32 field as a zigzag varint. fn thrift_write_field_i32(w: &mut W, last: &mut i16, id: i16, v: i32) { thrift_write_field_header(w, last, id, TC_I32); diff --git a/rust/src/models/readers/chunked/parquet.rs b/rust/src/models/readers/chunked/parquet.rs index 6b80552..61a4505 100644 --- a/rust/src/models/readers/chunked/parquet.rs +++ b/rust/src/models/readers/chunked/parquet.rs @@ -156,7 +156,7 @@ mod tests { } #[test] - fn categorical_column_roundtrips_via_dictionary_page() { + fn categorical_column_reads_back_as_utf8_strings() { // Regression: the reader's DictionaryPageHeader inner parser used // to consume raw i32 values without the Thrift type/id prefixes // the writer emits. That left the cursor 3 bytes shy of the @@ -164,11 +164,15 @@ mod tests { // length prefix was parsed as garbage - causing // `parse_dictionary_values` to ask for ~50 MB and fail with // `UnexpectedEof`. This test exercises the round-trip end-to-end. + // + // Parquet has no dictionary type, so the categorical column is a + // UTF8 string column with dictionary-encoded pages and reads back + // as strings. use crate::models::readers::parquet::load_parquet_table; use crate::models::writers::parquet::write_parquet_table; use minarrow::{ - Array, ArrowType, Bitmask, Buffer, CategoricalArray, Field, FieldArray, TextArray, - Vec64, ffi::arrow_dtype::CategoricalIndexType, + Array, ArrowType, Bitmask, Buffer, CategoricalArray, Field, FieldArray, MaskedArray, + TextArray, Vec64, ffi::arrow_dtype::CategoricalIndexType, }; use std::sync::Arc; @@ -228,6 +232,15 @@ mod tests { .expect("categorical column must round-trip via Parquet"); assert_eq!(got.n_rows, n_rows); assert_eq!(got.cols.len(), 1); + assert_eq!(got.cols[0].field.dtype, ArrowType::String); + match &got.cols[0].array { + Array::TextArray(TextArray::String32(strings)) => { + let expected: Vec> = + (0..n_rows).map(|i| Some(["red", "green", "blue"][i % 3])).collect(); + assert_eq!((0..n_rows).map(|i| strings.get(i)).collect::>(), expected); + } + other => panic!("expected a String column, got {other:?}"), + } let _ = std::fs::remove_file(&path); } diff --git a/rust/src/models/readers/parquet.rs b/rust/src/models/readers/parquet.rs index 4e8bc42..c461e20 100644 --- a/rust/src/models/readers/parquet.rs +++ b/rust/src/models/readers/parquet.rs @@ -6,10 +6,15 @@ //! # Parquet table reader - *reads into `minarrow::Table`* //! +//! Reads flat-schema Parquet files written by lightstream or by other +//! writers such as parquet-cpp (pyarrow) into a fully materialised Table. +//! //! ## Features -//! - Supports DataPageV2 plus the legacy V1 Parquet layout -//! - Decodes hybrid RLE/bit-packed definition levels and dictionary indices -//! - Handles `PLAIN` and `RLE_DICTIONARY` value encodings +//! - DataPageV1 and DataPageV2 page layouts, across any number of row groups +//! - Hybrid RLE/bit-packed definition levels and dictionary indices +//! - `PLAIN`, `RLE_DICTIONARY` and `PLAIN_DICTIONARY` value encodings, plus +//! `RLE` booleans. Dictionary-encoded columns of any physical type are +//! expanded on read. //! - Optional feature-gated Snappy / Zstd compression //! - Type maps to Arrow/Minarrow - {i32, i64, u32, u64, f32, f64, bool, utf8 //! dictionary Option { + if let Some(logical) = &se.logical_type { + return Some(logical.clone()); + } + #[cfg(feature = "decimal")] + if se.converted_type == Some(5) { + let precision = se.precision.unwrap_or(0) as u8; + let scale = se.scale.unwrap_or(0) as i8; + return Some(ParquetLogicalType::Decimal { precision, scale }); + } + ParquetLogicalType::from_converted_type(se.converted_type) +} + +/// Decode a FIXED_LEN_BYTE_ARRAY decimal buffer into unscaled values. +/// +/// Each value is `width` bytes of big-endian two's complement, up to 16 +/// bytes. Values are sign-extended into `i128` and then narrowed to the +/// decimal width the schema's precision selects. +#[cfg(feature = "decimal")] +fn decode_decimal_fixed_plain>( + buf: &[u8], + width: usize, +) -> Result, IoError> { + if width == 0 || width > 16 { + return Err(IoError::Format(format!( + "decimal FIXED_LEN_BYTE_ARRAY width {width} is outside 1..=16" + ))); + } + if buf.len() % width != 0 { + return Err(IoError::Format(format!( + "decimal value section of {} bytes is not a multiple of width {width}", + buf.len() + ))); + } + buf.chunks_exact(width) + .map(|chunk| { + let fill = if chunk[0] & 0x80 != 0 { 0xff } else { 0x00 }; + let mut bytes = [fill; 16]; + bytes[16 - width..].copy_from_slice(chunk); + T::try_from(i128::from_be_bytes(bytes)).map_err(|_| { + IoError::Format("decimal value exceeds the width its precision allows".into()) + }) + }) + .collect() +} /// Read an entire in-memory Table from a Parquet v2 file. /// @@ -63,27 +130,6 @@ pub fn load_parquet_table(r: R) -> Result { read_parquet_impl(r, None) } -/// Index type assumed for dictionary-encoded columns whose original -/// Arrow type was lost in the Parquet schema. -/// -/// The writer maps every `ArrowType::Dictionary(_)` to physical Int32 -/// with `NoneType` logical type, so the schema doesn't carry the index -/// width. Reads pick whichever width is the build's default categorical -/// type. Round-tripping a column written with `default_categorical_8` -/// disabled into a build that has it enabled (or vice versa) is not -/// supported. -#[inline] -fn default_categorical_index_type() -> CategoricalIndexType { - #[cfg(feature = "default_categorical_8")] - { - CategoricalIndexType::UInt8 - } - #[cfg(not(feature = "default_categorical_8"))] - { - CategoricalIndexType::UInt32 - } -} - /// Read only the named columns from a Parquet v2 file. /// /// Column names must match the schema's `path_in_schema` entries. Returns @@ -96,17 +142,31 @@ pub fn load_parquet_table_cols( r: R, columns: &[&str], ) -> Result { - let projection: std::collections::HashSet = + let projection: HashSet = columns.iter().map(|s| s.to_string()).collect(); read_parquet_impl(r, Some(projection)) } /// Shared implementation for full and projected Parquet reads. -/// When `projection` is `Some`, only columns whose names appear in the set -/// are read. Skipped columns never hit disk. +/// +/// Every row group contributes its column chunks to the same output +/// columns, so a file with several row groups reads as one Table. When +/// `projection` is `Some`, only columns whose names appear in the set are +/// read. Skipped columns never hit disk. +/// +/// ## Behaviour +/// - Values arrive in the Parquet layout, where a page's value section +/// holds non-null entries only. Each page is decoded to PLAIN entries and +/// scattered against its definition levels, so [`decode_column`] always +/// receives one entry per row. +/// - Dictionary-encoded pages of any physical type are expanded through +/// the row group's dictionary, so dictionary encoding stays a storage +/// detail of the writer and the column keeps its schema type. +/// - Repeated (nested) columns and unknown compression codecs are +/// reported as errors rather than decoded. fn read_parquet_impl( mut r: R, - projection: Option>, + projection: Option>, ) -> Result { // read the 8-byte footer r.seek(SeekFrom::End(-8))?; @@ -121,13 +181,22 @@ fn read_parquet_impl( r.seek(SeekFrom::End(-8 - footer_len as i64))?; let mut footer = vec![0u8; footer_len as usize]; r.read_exact(&mut footer)?; - let mut cur = std::io::Cursor::new(&footer); + let mut cur = Cursor::new(&footer); let meta = parse_file_metadata(&mut cur)?; + // Leaf schema elements carry a physical type and the root group element + // does not. Leaves sit in schema order, which is also the order of the + // column chunks inside every row group. + let leaves: Vec<(&SchemaElement, ParquetPhysicalType)> = meta + .schema + .iter() + .filter_map(|se| se.type_.map(|ty| (se, ty))) + .collect(); + // Validate projection names against schema before reading any data if let Some(ref proj) = projection { for name in proj { - if !meta.schema.iter().any(|se| se.name == *name) { + if !leaves.iter().any(|(se, _)| se.name == *name) { return Err(IoError::Format(format!( "column '{}' not found in schema", name @@ -136,117 +205,194 @@ fn read_parquet_impl( } } - // map Parquet schema -> Arrow types. The schema list opens with a root - // group element carrying no physical type; leaf column elements follow. - // Map only the leaves, in schema order, so indices line up with the row - // group's column chunks. - let arrow_types: Vec<_> = meta - .schema + if meta.row_groups.is_empty() { + return Err(IoError::Format("file has no row groups".into())); + } + if let Some(rg) = meta + .row_groups .iter() - .filter_map(|se| se.type_.map(|ty| (ty, se.converted_type))) - .map(|(ty, converted)| { - parquet_to_arrow_type(ty, ParquetLogicalType::from_converted_type(converted)) - }) - .collect::>()?; - - // single row-group, flat schema only - let rg = &meta.row_groups[0]; - let mut columns = Vec::with_capacity(rg.columns.len()); + .find(|rg| rg.columns.len() != leaves.len()) + { + return Err(IoError::Format(format!( + "row group has {} column chunks for {} schema leaves", + rg.columns.len(), + leaves.len() + ))); + } - for (col_idx, chunk) in rg.columns.iter().enumerate() { - let cmeta = &chunk.meta_data; - let col_name = &cmeta.path_in_schema[0]; + let mut columns = Vec::with_capacity(leaves.len()); + for (col_idx, &(leaf, physical)) in leaves.iter().enumerate() { // Skip columns not in the projection if let Some(ref proj) = projection - && !proj.contains(col_name) + && !proj.contains(&leaf.name) { continue; } - // The Parquet schema has no slot to record that a column was - // originally a Dictionary - the writer maps Dictionary(_) to a - // physical Int32 with no logical type. Recover the Arrow shape - // from the column metadata: a column carrying a dictionary page - // is decoded as Dictionary using the build's default index - // width (UInt8 when `default_categorical_8` is on, UInt32 - // otherwise). Round-trip across feature flags isn't supported. - let schema_ty = &arrow_types[col_idx]; - let dictionary_ty = if cmeta.dictionary_page_offset.is_some() { - Some(ArrowType::Dictionary(default_categorical_index_type())) - } else { - None - }; - let ty: &ArrowType = dictionary_ty.as_ref().unwrap_or(schema_ty); - - // read the DICTIONARY_PAGE if present - let dict = if let Some(dict_off) = cmeta.dictionary_page_offset { - r.seek(SeekFrom::Start(dict_off as u64))?; - let ph = parse_page_header(&mut r)?; - if ph.type_ != PageType::DictionaryPage { - return Err(IoError::Format("expected DICTIONARY_PAGE".into())); - } - let mut compr = vec![0u8; ph.compressed_page_size as usize]; - r.read_exact(&mut compr)?; - match map_codec(cmeta.codec) { - Some(c) => parse_dictionary_values(&decompress(&compr, c)?)?, - None => parse_dictionary_values(&compr)?, + // OPTIONAL leaves carry one definition level per row. REQUIRED + // leaves carry none. REPEATED leaves describe nested data. + let max_def_level: u8 = match leaf.repetition_type { + 0 => 0, + 1 => 1, + other => { + return Err(IoError::UnsupportedType(format!( + "repeated column '{}' (repetition_type {other})", + leaf.name + ))); } - } else { - Vec::new() }; - // walk all the DATA_PAGE_V2s in a row - let total_vals = cmeta.num_values as usize; - let mut def_levels = Vec::with_capacity(total_vals); - let mut values_buf = Vec::new(); - let mut pages_read = 0; - let mut page_encoding = ParquetEncoding::Plain; - - // seek once to the first data page - r.seek(SeekFrom::Start(cmeta.data_page_offset as u64))?; - - while pages_read < total_vals { - // parse the next page header - let ph = parse_page_header(&mut r)?; - let (page_defs, enc, page_vals) = match ph.type_ { - PageType::DataPageV2 => read_data_page_v2(&mut r, &ph, cmeta)?, - PageType::DataPage => read_data_page_v1(&mut r, &ph, cmeta)?, - t => return Err(IoError::Format(format!("unsupported page type {:?}", t))), - }; - if pages_read == 0 { - page_encoding = enc; + let ty = parquet_to_arrow_type(physical, logical_type_from_schema(leaf))?; + let layout = ValueLayout::of(physical, leaf.type_length)?; + + let mut def_levels: Vec = Vec::new(); + let mut values: Vec = Vec::new(); + + for rg in &meta.row_groups { + let cmeta = &rg.columns[col_idx].meta_data; + let codec = map_codec(cmeta.codec)?; + + // The row group's dictionary entries without their PLAIN length + // prefix. + let mut dict_entries: Vec> = Vec::new(); + if let Some(dict_off) = cmeta.dictionary_page_offset { + r.seek(SeekFrom::Start(dict_off as u64))?; + let ph = parse_page_header(&mut r)?; + if ph.type_ != PageType::DictionaryPage { + return Err(IoError::Format("expected DICTIONARY_PAGE".into())); + } + let mut body = vec![0u8; ph.compressed_page_size as usize]; + r.read_exact(&mut body)?; + let body = match codec { + Some(c) => decompress(&body, c)?, + None => body, + }; + { + dict_entries = match layout { + ValueLayout::LengthPrefixed => parse_dictionary_values(&body)?, + ValueLayout::Fixed(width) => { + if body.len() % width != 0 { + return Err(IoError::Format(format!( + "dictionary page of {} bytes is not a multiple of the {width}-byte value width", + body.len() + ))); + } + body.chunks_exact(width).map(<[u8]>::to_vec).collect() + } + }; + } } - // accumulate `page_defs.len()` logical rows - let this_count = page_defs.len().min(total_vals - pages_read); - def_levels.extend_from_slice(&page_defs[..this_count]); - values_buf.extend_from_slice(&page_vals); - pages_read += this_count; + // Walk the chunk's data pages. num_values counts every row of + // the chunk, nulls included, which is the definition level + // count. + r.seek(SeekFrom::Start(cmeta.data_page_offset as u64))?; + let chunk_levels = cmeta.num_values as usize; + let mut levels_read = 0usize; + + while levels_read < chunk_levels { + let ph = parse_page_header(&mut r)?; + let (page_defs, encoding, page_values) = match ph.type_ { + PageType::DataPage => read_data_page_v1(&mut r, &ph, codec, max_def_level)?, + PageType::DataPageV2 => { + read_data_page_v2(&mut r, &ph, codec, max_def_level)? + } + t => return Err(IoError::Format(format!("unsupported page type {:?}", t))), + }; + let n_valid = page_defs.iter().filter(|&&v| v).count(); + + // PLAIN entries for the page's non-null values. Boolean pages + // unpack to one byte per value here so every physical type + // scatters through the same fixed or length-prefixed layout. + let plain: Vec = match (encoding, physical) { + (ParquetEncoding::Plain, ParquetPhysicalType::Boolean) => { + if page_values.len() * 8 < n_valid { + return Err(IoError::Format("truncated boolean page".into())); + } + (0..n_valid) + .map(|i| (page_values[i / 8] >> (i % 8)) & 1) + .collect() + } + (ParquetEncoding::Rle, ParquetPhysicalType::Boolean) => { + // RLE booleans carry a 4-byte run length ahead of + // the hybrid run, like V1 levels. + let run = length_prefixed_run(&page_values)?; + decode_hybrid(run, 1, n_valid)? + .iter() + .map(|&v| v as u8) + .collect() + } + (ParquetEncoding::Plain, _) => page_values, + (ParquetEncoding::RleDictionary | ParquetEncoding::PlainDictionary, _) => { + let indices = if n_valid == 0 { + Vec64::new() + } else { + decode_dictionary_indices_rle(&page_values, n_valid)? + }; + let mut plain = Vec::new(); + for &idx in indices.iter() { + let entry = dict_entries.get(idx as usize).ok_or_else(|| { + IoError::Format(format!("dictionary index {idx} out of range")) + })?; + if layout == ValueLayout::LengthPrefixed { + plain.extend_from_slice(&(entry.len() as u32).to_le_bytes()); + } + plain.extend_from_slice(entry); + } + plain + } + (enc, _) => { + return Err(IoError::UnsupportedEncoding(format!( + "{enc:?} for column '{}'", + leaf.name + ))); + } + }; - // cursor is at the start of the next page header + scatter_by_definition_levels(&mut values, &plain, &page_defs, layout)?; + levels_read += page_defs.len(); + def_levels.extend(page_defs); + } } - // decode the column array - let array = decode_column( - ty, - page_encoding, - &dict, - &values_buf, - total_vals, - def_levels.clone(), - )?; + if def_levels.len() != meta.num_rows as usize { + return Err(IoError::Format(format!( + "column '{}' holds {} rows, file metadata says {}", + leaf.name, + def_levels.len(), + meta.num_rows + ))); + } + + // UTF8 columns use 32-bit offsets unless the data outgrows them. + #[cfg(feature = "large_string")] + let ty = if ty == ArrowType::String && values.len() > u32::MAX as usize { + ArrowType::LargeString + } else { + ty + }; + #[cfg(not(feature = "large_string"))] + if ty == ArrowType::String && values.len() > u32::MAX as usize { + return Err(IoError::UnsupportedType(format!( + "column '{}' holds more than 4 GiB of string data, which needs the large_string feature", + leaf.name + ))); + } + + let null_count = def_levels.iter().filter(|&&b| !b).count(); + let array = decode_column(&ty, physical, layout, &values, def_levels.len(), def_levels)?; columns.push(FieldArray { field: Field { - name: col_name.clone(), - dtype: ty.clone(), - nullable: chunk.meta_data.definition_level >= 1 || def_levels.iter().any(|&b| !b), + name: leaf.name.clone(), + dtype: ty, + nullable: max_def_level > 0, metadata: Default::default(), } .into(), array, - null_count: def_levels.iter().filter(|&&b| !b).count(), + null_count, }); } @@ -258,24 +404,172 @@ fn read_parquet_impl( }) } -/// DataPageV2 reader: read exactly `compressed_page_size` bytes, split into -/// rep / def, decompress the remainder, decode def‐levels, return the raw -/// values and the page encoding. +/// Error for a DECIMAL column stored in a physical type the reader does +/// not decode, which is BYTE_ARRAY. +#[cfg(feature = "decimal")] +fn decimal_storage_error(physical: ParquetPhysicalType) -> IoError { + IoError::UnsupportedType(format!("DECIMAL stored as {physical:?}")) +} + +/// Byte layout of one value inside a PLAIN value section. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum ValueLayout { + /// Every value occupies the given number of bytes. + Fixed(usize), + /// A 4-byte little-endian length precedes each value's bytes. + LengthPrefixed, +} + +impl ValueLayout { + /// Layout of the physical type once a page has been decoded to PLAIN + /// entries. Booleans count as one byte per value because the page + /// readers unpack them before scattering. `type_length` is the schema + /// element's width for FIXED_LEN_BYTE_ARRAY leaves. + fn of(physical: ParquetPhysicalType, type_length: Option) -> Result { + Ok(match physical { + ParquetPhysicalType::Boolean => ValueLayout::Fixed(1), + ParquetPhysicalType::Int32 | ParquetPhysicalType::Float => ValueLayout::Fixed(4), + ParquetPhysicalType::Int64 | ParquetPhysicalType::Double => ValueLayout::Fixed(8), + ParquetPhysicalType::ByteArray => ValueLayout::LengthPrefixed, + ParquetPhysicalType::FixedLenByteArray => match type_length { + Some(width) if width > 0 => ValueLayout::Fixed(width as usize), + _ => { + return Err(IoError::Format( + "FIXED_LEN_BYTE_ARRAY leaf without a type_length".into(), + )); + } + }, + }) + } +} + +/// Append a page's PLAIN entries to `out` with a zero entry at every null +/// row, so the column buffer holds one entry per row. +/// +/// Parquet value sections hold non-null values only, while the PLAIN +/// decoders behind [`decode_column`] expect one entry per row with the null +/// mask applied afterwards. A null row becomes zero bytes for a fixed-width +/// value or a zero length for a length-prefixed value. +fn scatter_by_definition_levels( + out: &mut Vec, + plain: &[u8], + def_levels: &[bool], + layout: ValueLayout, +) -> Result<(), IoError> { + if def_levels.iter().all(|&v| v) { + out.extend_from_slice(plain); + return Ok(()); + } + let mut pos = 0usize; + for &valid in def_levels { + if !valid { + match layout { + ValueLayout::Fixed(width) => out.extend(std::iter::repeat_n(0u8, width)), + ValueLayout::LengthPrefixed => out.extend_from_slice(&[0u8; 4]), + } + continue; + } + let width = match layout { + ValueLayout::Fixed(width) => width, + ValueLayout::LengthPrefixed => { + let len = plain + .get(pos..pos + 4) + .map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize) + .ok_or_else(|| { + IoError::Format("value section shorter than its definition levels".into()) + })?; + 4 + len + } + }; + let entry = plain.get(pos..pos + width).ok_or_else(|| { + IoError::Format("value section shorter than its definition levels".into()) + })?; + out.extend_from_slice(entry); + pos += width; + } + Ok(()) +} + +/// Split a 4-byte little-endian length off the front of a level or RLE +/// boolean stream and return the run it introduces. +fn length_prefixed_run(buf: &[u8]) -> Result<&[u8], IoError> { + let len = buf + .get(..4) + .map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize) + .ok_or_else(|| IoError::Format("truncated length-prefixed run".into()))?; + buf.get(4..4 + len) + .ok_or_else(|| IoError::Format("truncated length-prefixed run".into())) +} + +/// DataPageV1 reader. +/// +/// The whole page body is compressed as one unit. Inside it, repetition +/// levels (absent for a flat schema) and definition levels (absent for a +/// REQUIRED column) each lead with a 4-byte length, and the values follow. +/// Returns the definition levels, the value encoding and the raw values. +fn read_data_page_v1( + r: &mut R, + ph: &PageHeader, + codec: Option, + max_def_level: u8, +) -> Result<(Vec, ParquetEncoding, Vec), IoError> { + let h = ph + .data_page_header + .as_ref() + .ok_or_else(|| IoError::Format("missing DataPageHeader".into()))?; + + let mut body = vec![0u8; ph.compressed_page_size as usize]; + r.read_exact(&mut body)?; + let body = match codec { + Some(c) => decompress(&body, c)?, + None => body, + }; + + let n = h.num_values as usize; + let mut pos = 0usize; + let def_levels = if max_def_level == 0 { + vec![true; n] + } else { + if h.definition_level_encoding != ParquetEncoding::Rle { + return Err(IoError::UnsupportedEncoding(format!( + "{:?} definition levels", + h.definition_level_encoding + ))); + } + let run = length_prefixed_run(&body[pos..])?; + pos += 4 + run.len(); + decode_hybrid(run, max_def_level, n)? + .into_iter() + .map(|v| v == max_def_level as u32) + .collect() + }; + + Ok((def_levels, h.encoding, body[pos..].to_vec())) +} + +/// DataPageV2 reader. +/// +/// Repetition and definition levels sit uncompressed at the front of the +/// page with their byte lengths in the header. The value section that +/// follows is compressed only when the header says so. Returns the +/// definition levels, the value encoding and the raw values. fn read_data_page_v2( r: &mut R, ph: &PageHeader, - cmeta: &ColumnMetadata, + codec: Option, + max_def_level: u8, ) -> Result<(Vec, ParquetEncoding, Vec), IoError> { let h = ph .data_page_header_v2 .as_ref() .ok_or_else(|| IoError::Format("missing DataPageHeaderV2".into()))?; - // 1) consume repetition‐levels bytes + // 1) consume repetition-levels bytes. A flat schema has no repetition + // levels, though a writer may still emit a run for them. let mut rep = vec![0u8; h.repetition_levels_byte_length as usize]; r.read_exact(&mut rep)?; - // 2) consume definition‐levels bytes + // 2) consume definition-levels bytes let mut def = vec![0u8; h.definition_levels_byte_length as usize]; r.read_exact(&mut def)?; @@ -287,55 +581,39 @@ fn read_data_page_v2( r.read_exact(&mut vs)?; // 4) decompress if needed - let values_raw = match (h.is_compressed, map_codec(cmeta.codec)) { + let values_raw = match (h.is_compressed, codec) { (true, Some(c)) => decompress(&vs, c)?, _ => vs, }; - // 5) decode definition‐levels (we ignore repetition entirely) - let def_levels = if cmeta.definition_level == 0 && def.is_empty() { - vec![true; h.num_rows as usize] + // 5) decode definition levels. num_values counts every row of the + // page, nulls included, which is the definition level count. + let n = h.num_values as usize; + let def_levels = if max_def_level == 0 || def.is_empty() { + vec![true; n] } else { - decode_hybrid(&def, 1, h.num_rows as usize)? + decode_hybrid(&def, max_def_level, n)? .into_iter() - .map(|v| v != 0) + .map(|v| v == max_def_level as u32) .collect() }; Ok((def_levels, h.encoding, values_raw)) } -/// DataPageV1 reader -fn read_data_page_v1( - r: &mut R, - _ph: &PageHeader, - cmeta: &ColumnMetadata, -) -> Result<(Vec, ParquetEncoding, Vec), IoError> { - // read the 4-byte prefix of the def‐levels stream - let def = read_len_prefixed(r)?; - // read the remaining compressed values for this page, - // which in V1 is “to the end of this page” - as V1 only writes one page - let mut vs = Vec::new(); - r.read_to_end(&mut vs)?; - - let num_vals = (cmeta.num_values as usize).max(def_levels_count(&def, 1)); - let def_levels = if cmeta.definition_level == 0 && def.is_empty() { - vec![true; num_vals] - } else { - decode_hybrid(&def, 1, num_vals)? - .into_iter() - .map(|v| v != 0) - .collect() - }; - Ok((def_levels, ParquetEncoding::Plain, vs)) -} - // Column-value decoder +/// Build the column array from PLAIN entries, one per row. +/// +/// `buf` holds one entry per row in the layout the PLAIN decoders expect, +/// with a zero entry under each null. `physical` and `layout` describe the +/// entries for types with more than one storage form, which for now means +/// decimals stored as INT32, INT64 or FIXED_LEN_BYTE_ARRAY. +#[cfg_attr(not(feature = "decimal"), allow(unused_variables))] fn decode_column( ty: &ArrowType, - enc: ParquetEncoding, - dict: &[Vec], + physical: ParquetPhysicalType, + layout: ValueLayout, buf: &[u8], len: usize, def_levels: Vec, @@ -344,50 +622,33 @@ fn decode_column( Ok(match ty { // numerics - ArrowType::Int32 if enc == ParquetEncoding::Plain => { - Array::NumericArray(NumericArray::Int32(Arc::new(IntegerArray::from_vec64( - decode_int32_plain(buf)?, - mask, - )))) - } - ArrowType::UInt32 if enc == ParquetEncoding::Plain => { - Array::NumericArray(NumericArray::UInt32(Arc::new(IntegerArray::from_vec64( - decode_uint32_as_int32_plain(buf)?, - mask, - )))) - } - ArrowType::Int64 if enc == ParquetEncoding::Plain => { - Array::NumericArray(NumericArray::Int64(Arc::new(IntegerArray::from_vec64( - decode_int64_plain(buf)?, - mask, - )))) - } - ArrowType::UInt64 if enc == ParquetEncoding::Plain => { - Array::NumericArray(NumericArray::UInt64(Arc::new(IntegerArray::from_vec64( - decode_uint64_as_int64_plain(buf)?, - mask, - )))) - } - ArrowType::Float32 if enc == ParquetEncoding::Plain => { - Array::NumericArray(NumericArray::Float32(Arc::new(FloatArray::from_vec64( - decode_float32_plain(buf)?, - mask, - )))) - } - ArrowType::Float64 if enc == ParquetEncoding::Plain => { - Array::NumericArray(NumericArray::Float64(Arc::new(FloatArray::from_vec64( - decode_float64_plain(buf)?, - mask, - )))) - } - - // booleans - ArrowType::Boolean if enc == ParquetEncoding::Plain => { - Array::BooleanArray(Arc::new(BooleanArray::new(Bitmask::from_bytes(buf, len), mask))) + ArrowType::Int32 => Array::NumericArray(NumericArray::Int32(Arc::new( + IntegerArray::from_vec64(decode_int32_plain(buf)?, mask), + ))), + ArrowType::UInt32 => Array::NumericArray(NumericArray::UInt32(Arc::new( + IntegerArray::from_vec64(decode_uint32_as_int32_plain(buf)?, mask), + ))), + ArrowType::Int64 => Array::NumericArray(NumericArray::Int64(Arc::new( + IntegerArray::from_vec64(decode_int64_plain(buf)?, mask), + ))), + ArrowType::UInt64 => Array::NumericArray(NumericArray::UInt64(Arc::new( + IntegerArray::from_vec64(decode_uint64_as_int64_plain(buf)?, mask), + ))), + ArrowType::Float32 => Array::NumericArray(NumericArray::Float32(Arc::new( + FloatArray::from_vec64(decode_float32_plain(buf)?, mask), + ))), + ArrowType::Float64 => Array::NumericArray(NumericArray::Float64(Arc::new( + FloatArray::from_vec64(decode_float64_plain(buf)?, mask), + ))), + + // booleans, one byte per row + ArrowType::Boolean => { + let bits: Vec = buf.iter().map(|&b| b != 0).collect(); + Array::BooleanArray(Arc::new(BooleanArray::new(Bitmask::from_bools(&bits), mask))) } // strings - ArrowType::String if enc == ParquetEncoding::Plain => { + ArrowType::String => { let (offsets, data) = decode_string_plain(buf, len)?; Array::TextArray(TextArray::String32(Arc::new(StringArray { offsets: offsets.into(), @@ -396,7 +657,7 @@ fn decode_column( }))) } #[cfg(feature = "large_string")] - ArrowType::LargeString if enc == ParquetEncoding::Plain => { + ArrowType::LargeString => { use crate::models::decoders::parquet::decode_large_string_plain; let (offsets, data) = decode_large_string_plain(buf, len)?; @@ -407,135 +668,87 @@ fn decode_column( }))) } - // dictionary / categoricals - ArrowType::Dictionary(key_ty) => { - match (key_ty, enc) { - // u32 keys - #[cfg(any( - not(feature = "default_categorical_8"), - feature = "extended_categorical" - ))] - (CategoricalIndexType::UInt32, ParquetEncoding::RleDictionary) => { - let idx = decode_dictionary_indices_rle(buf, len)?; - build_cat32(idx, dict, mask) - } - #[cfg(any( - not(feature = "default_categorical_8"), - feature = "extended_categorical" - ))] - (CategoricalIndexType::UInt32, ParquetEncoding::Plain) => { - let idx = decode_uint32_as_int32_plain(buf)?; - build_cat32(idx, dict, mask) - } - - // u8 keys (default when default_categorical_8 is enabled) - #[cfg(feature = "default_categorical_8")] - (CategoricalIndexType::UInt8, ParquetEncoding::RleDictionary) => { - let idx = decode_dictionary_indices_rle(buf, len)?; - build_cat8(idx, dict, mask) - } - #[cfg(feature = "default_categorical_8")] - (CategoricalIndexType::UInt8, ParquetEncoding::Plain) => { - let idx = decode_uint32_as_int32_plain(buf)?; - build_cat8(idx, dict, mask) - } - - // optional u64 keys - #[cfg(all(feature = "extended_categorical", feature = "large_string"))] - (CategoricalIndexType::UInt64, ParquetEncoding::RleDictionary) => { - let idx = decode_dictionary_indices_rle(buf, len)?; - let idx = idx.into_iter().map(|v| v as u64).collect(); - build_cat64(idx, dict, mask) - } - #[cfg(all(feature = "extended_categorical", feature = "large_string"))] - (CategoricalIndexType::UInt64, ParquetEncoding::Plain) => { - let idx = decode_uint64_as_int64_plain(buf)?; - build_cat64(idx, dict, mask) - } - - _ => { - return Err(IoError::UnsupportedEncoding(format!( - "{:?} + {:?}", - key_ty, enc - ))); - } - } - } - - // temporal + // temporal. DATE and TIME(MILLIS) are INT32, every other unit INT64. #[cfg(feature = "datetime")] - ArrowType::Date32 if enc == ParquetEncoding::Plain => { - Array::TemporalArray(TemporalArray::Datetime32(Arc::new(DatetimeArray { + ArrowType::Date32 => Array::TemporalArray(TemporalArray::Datetime32(Arc::new( + DatetimeArray { data: decode_datetime32_plain(buf)?.into(), null_mask: mask, - time_unit: Default::default(), - }))) - } + time_unit: TimeUnit::Days, + }, + ))), + #[cfg(feature = "datetime")] + ArrowType::Time32(unit) => Array::TemporalArray(TemporalArray::Datetime32(Arc::new( + DatetimeArray { + data: decode_datetime32_plain(buf)?.into(), + null_mask: mask, + time_unit: unit.clone(), + }, + ))), #[cfg(feature = "datetime")] - ArrowType::Date64 if enc == ParquetEncoding::Plain => { + ArrowType::Timestamp(unit, _) | ArrowType::Time64(unit) => { Array::TemporalArray(TemporalArray::Datetime64(Arc::new(DatetimeArray { data: decode_datetime64_plain(buf)?.into(), null_mask: mask, - time_unit: Default::default(), + time_unit: unit.clone(), + }))) + } + + // decimals + #[cfg(feature = "decimal")] + ArrowType::Decimal32(precision, scale) => { + let data = match (physical, layout) { + (ParquetPhysicalType::Int32, _) => decode_int32_plain(buf)?, + (ParquetPhysicalType::FixedLenByteArray, ValueLayout::Fixed(width)) => { + decode_decimal_fixed_plain::(buf, width)? + } + _ => return Err(decimal_storage_error(physical)), + }; + Array::NumericArray(NumericArray::Decimal32(Arc::new(DecimalArray { + data: data.into(), + null_mask: mask, + precision: *precision, + scale: *scale, + }))) + } + #[cfg(feature = "decimal")] + ArrowType::Decimal64(precision, scale) => { + let data = match (physical, layout) { + (ParquetPhysicalType::Int64, _) => decode_int64_plain(buf)?, + (ParquetPhysicalType::FixedLenByteArray, ValueLayout::Fixed(width)) => { + decode_decimal_fixed_plain::(buf, width)? + } + _ => return Err(decimal_storage_error(physical)), + }; + Array::NumericArray(NumericArray::Decimal64(Arc::new(DecimalArray { + data: data.into(), + null_mask: mask, + precision: *precision, + scale: *scale, + }))) + } + #[cfg(feature = "decimal")] + ArrowType::Decimal128(precision, scale) => { + let data = match (physical, layout) { + (ParquetPhysicalType::FixedLenByteArray, ValueLayout::Fixed(width)) => { + decode_decimal_fixed_plain::(buf, width)? + } + _ => return Err(decimal_storage_error(physical)), + }; + Array::NumericArray(NumericArray::Decimal128(Arc::new(DecimalArray { + data: data.into(), + null_mask: mask, + precision: *precision, + scale: *scale, }))) } _ => { - return Err(IoError::UnsupportedType(format!( - "decode {:?} / {:?}", - ty, enc - ))); + return Err(IoError::UnsupportedType(format!("decode {:?}", ty))); } }) } -// categorical builders - -#[cfg(any( - not(feature = "default_categorical_8"), - feature = "extended_categorical" -))] -fn build_cat32(idx: Vec64, dict_raw: &[Vec], mask: Option) -> Array { - let dict = dict_raw - .iter() - .map(|b| String::from_utf8(b.clone()).unwrap()) - .collect::>() - .into(); - Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray { - data: idx.into(), - unique_values: dict, - null_mask: mask, - }))) -} - -#[cfg(feature = "default_categorical_8")] -fn build_cat8(idx: Vec64, dict_raw: &[Vec], mask: Option) -> Array { - let dict = dict_raw - .iter() - .map(|b| String::from_utf8(b.clone()).unwrap()) - .collect::>(); - let idx8: Vec64 = idx.iter().map(|&v| v as u8).collect(); - Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray { - data: idx8.into(), - unique_values: dict, - null_mask: mask, - }))) -} - -#[cfg(all(feature = "extended_categorical", feature = "large_string"))] -fn build_cat64(idx: Vec64, dict_raw: &[Vec], mask: Option) -> Array { - let dict = dict_raw - .iter() - .map(|b| String::from_utf8(b.clone()).unwrap()) - .collect::>() - .into(); - Array::TextArray(TextArray::Categorical64(Arc::new(CategoricalArray { - data: idx.into(), - unique_values: dict, - null_mask: mask, - }))) -} - // RLE/bit-packed Hybrid decoder fn decode_hybrid(buf: &[u8], bit_width: u8, n: usize) -> Result, IoError> { @@ -607,41 +820,27 @@ fn read_uleb128(buf: &[u8]) -> Result<(u64, usize), IoError> { Err(IoError::Format("ULEB128 overflow/truncate".into())) } -// utility for legacy V1 count heuristic -fn def_levels_count(buf: &[u8], bw: u8) -> usize { - if buf.is_empty() { - 0 - } else if buf[0] & 1 == 0 { - ((buf[0] as usize) >> 1).min(1 << bw) - } else { - 0 - } -} - // Misc helpers -fn map_codec(id: i32) -> Option { +/// Resolve the column chunk's codec id. Uncompressed is `None`. Codecs +/// outside the enabled compression features are an error, since their +/// pages cannot be decoded. +fn map_codec(id: i32) -> Result, IoError> { match id { - 0 => None, + 0 => Ok(None), #[cfg(feature = "snappy")] - 1 => Some(Compression::Snappy), + 1 => Ok(Some(Compression::Snappy)), #[cfg(feature = "zstd")] - 6 => Some(Compression::Zstd), // spec: ZSTD = 6 - _ => None, + 6 => Ok(Some(Compression::Zstd)), // spec: ZSTD = 6 + other => Err(IoError::Compression(format!( + "unsupported parquet compression codec {other}" + ))), } } -fn read_len_prefixed(r: &mut R) -> Result, IoError> { - let mut l4 = [0u8; 4]; - r.read_exact(&mut l4)?; - let len = u32::from_le_bytes(l4) as usize; - let mut buf = vec![0u8; len]; - r.read_exact(&mut buf)?; - Ok(buf) -} - +/// Split a PLAIN BYTE_ARRAY dictionary page body into its entries. fn parse_dictionary_values(buf: &[u8]) -> Result>, IoError> { - let mut c = std::io::Cursor::new(buf); + let mut c = Cursor::new(buf); let mut out = Vec::new(); while (c.position() as usize) < buf.len() { let mut l4 = [0u8; 4]; @@ -735,6 +934,7 @@ fn parse_schema_element(r: &mut R) -> Result { let mut scale = None; let mut field_id = None; let mut num_children = None; + let mut logical_type = None; loop { let (tpe, id) = thrift_read_field_begin(r, &mut last)?; @@ -743,10 +943,14 @@ fn parse_schema_element(r: &mut R) -> Result { } match id { 1 => { - type_ = Some( - ParquetPhysicalType::from_i32(thrift_read_i32(r)?) - .ok_or_else(|| IoError::Format("Invalid type_".into()))?, - ) + let v = thrift_read_i32(r)?; + type_ = Some(ParquetPhysicalType::from_i32(v).ok_or_else(|| { + if v == 3 { + IoError::UnsupportedType("INT96 columns are not supported".into()) + } else { + IoError::Format(format!("invalid physical type {v}")) + } + })?) } 2 => type_length = Some(thrift_read_i32(r)?), 3 => repetition_type = Some(thrift_read_i32(r)?), @@ -756,6 +960,7 @@ fn parse_schema_element(r: &mut R) -> Result { 7 => scale = Some(thrift_read_i32(r)?), 8 => precision = Some(thrift_read_i32(r)?), 9 => field_id = Some(thrift_read_i32(r)?), + 10 => logical_type = parse_logical_type(r)?, _ => thrift_skip_field(r, tpe)?, } } @@ -765,6 +970,7 @@ fn parse_schema_element(r: &mut R) -> Result { repetition_type: repetition_type.unwrap_or(0), type_, converted_type, + logical_type, type_length, precision, scale, @@ -773,6 +979,127 @@ fn parse_schema_element(r: &mut R) -> Result { }) } +/// Parse the `LogicalType` union of a schema element. +/// +/// The union holds one struct field whose id names the annotation: STRING +/// 1, DECIMAL 5, DATE 6, TIME 7, TIMESTAMP 8, INTEGER 10. TIME and +/// TIMESTAMP carry `isAdjustedToUTC` and a `TimeUnit` union of MILLIS 1, +/// MICROS 2 and NANOS 3. Annotations outside the supported set, such as +/// MAP, LIST, JSON and UUID, return `None` and the column falls back to +/// its converted type. +fn parse_logical_type(r: &mut R) -> Result, IoError> { + let mut last = 0i16; + let mut logical = None; + loop { + let (tpe, id) = thrift_read_field_begin(r, &mut last)?; + if tpe == 0 { + break; + } + if tpe != TC_STRUCT { + thrift_skip_field(r, tpe)?; + continue; + } + logical = match id { + 1 => { + thrift_skip_value(r, TC_STRUCT)?; + Some(ParquetLogicalType::Utf8) + } + #[cfg(feature = "decimal")] + 5 => { + let mut inner = 0i16; + let mut scale = 0i32; + let mut precision = 0i32; + loop { + let (t, f) = thrift_read_field_begin(r, &mut inner)?; + if t == 0 { + break; + } + match f { + 1 => scale = thrift_read_i32(r)?, + 2 => precision = thrift_read_i32(r)?, + _ => thrift_skip_field(r, t)?, + } + } + Some(ParquetLogicalType::Decimal { + precision: precision as u8, + scale: scale as i8, + }) + } + #[cfg(feature = "datetime")] + 6 => { + thrift_skip_value(r, TC_STRUCT)?; + Some(ParquetLogicalType::Date32) + } + #[cfg(feature = "datetime")] + 7 | 8 => { + let mut inner = 0i16; + let mut unit = 0i16; + let mut utc = false; + loop { + let (t, f) = thrift_read_field_begin(r, &mut inner)?; + if t == 0 { + break; + } + match f { + 1 if t == TC_BOOL_TRUE || t == TC_BOOL_FALSE => utc = t == TC_BOOL_TRUE, + 2 if t == TC_STRUCT => { + let mut unit_last = 0i16; + loop { + let (ut, uf) = thrift_read_field_begin(r, &mut unit_last)?; + if ut == 0 { + break; + } + unit = uf; + thrift_skip_field(r, ut)?; + } + } + _ => thrift_skip_field(r, t)?, + } + } + match (id, unit) { + (7, 1) => Some(ParquetLogicalType::TimeMillis), + (7, 2) => Some(ParquetLogicalType::TimeMicros), + (7, 3) => Some(ParquetLogicalType::TimeNanos), + (8, 1) => Some(ParquetLogicalType::TimestampMillis { utc }), + (8, 2) => Some(ParquetLogicalType::TimestampMicros { utc }), + (8, 3) => Some(ParquetLogicalType::TimestampNanos { utc }), + _ => None, + } + } + 10 => { + let mut inner = 0i16; + let mut bit_width = 0u8; + let mut is_signed = true; + loop { + let (t, f) = thrift_read_field_begin(r, &mut inner)?; + if t == 0 { + break; + } + match (f, t) { + (1, TC_BYTE) => { + let mut b = [0u8; 1]; + r.read_exact(&mut b)?; + bit_width = b[0]; + } + (2, TC_BOOL_TRUE) => is_signed = true, + (2, TC_BOOL_FALSE) => is_signed = false, + _ => thrift_skip_field(r, t)?, + } + } + Some(ParquetLogicalType::IntType { + bit_width, + is_signed, + }) + } + _ => { + thrift_skip_value(r, TC_STRUCT)?; + None + } + }; + } + Ok(logical) +} + fn parse_row_group(r: &mut R) -> Result { let mut last = 0i16; let mut columns = Vec::new(); @@ -889,7 +1216,6 @@ fn parse_column_meta_data(r: &mut R) -> Result data_page_offset: data_page_offset.unwrap_or(0), dictionary_page_offset, statistics, - definition_level: 0, }) } @@ -1274,11 +1600,6 @@ mod tests { use super::*; - /// Build a Vec string dictionary from &strs. - fn dict(strings: &[&str]) -> Vec> { - strings.iter().map(|s| s.as_bytes().to_vec()).collect() - } - #[test] fn hybrid_rle_run() { // pattern: 6× value 3, bit-width = 2 @@ -1309,65 +1630,6 @@ mod tests { let out = super::decode_hybrid(buf, bit_width, expect.len()).unwrap(); assert_eq!(out.as_slice(), expect.as_slice()); } - #[cfg(not(feature = "default_categorical_8"))] - #[test] - fn decode_column_categorical_rle_dictionary() { - let dict_raw = dict(&["foo", "bar"]); - let idx: Vec = vec![0, 1, 1, 0]; - let mut encoded = Vec::new(); - encode_dictionary_indices_rle(&idx, &mut encoded).unwrap(); - - let def_levels = vec![true; idx.len()]; - - let array = super::decode_column( - &ArrowType::Dictionary(CategoricalIndexType::UInt32), - ParquetEncoding::RleDictionary, - &dict_raw, - &encoded, - idx.len(), - def_levels, - ) - .expect("decode_column failed"); - - match array { - Array::TextArray(TextArray::Categorical32(cat)) => { - assert_eq!(cat.data.as_slice(), idx.as_slice()); - let uniq: Vec<_> = cat.unique_values.iter().collect(); - assert_eq!(uniq, vec!["foo", "bar"]); - } - _ => panic!("unexpected array variant {:?}", array), - } - } - - #[cfg(feature = "default_categorical_8")] - #[test] - fn decode_column_categorical_rle_dictionary() { - let dict_raw = dict(&["foo", "bar"]); - let idx: Vec = vec![0, 1, 1, 0]; - let mut encoded = Vec::new(); - encode_dictionary_indices_rle(&idx, &mut encoded).unwrap(); - - let def_levels = vec![true; idx.len()]; - - let array = super::decode_column( - &ArrowType::Dictionary(CategoricalIndexType::UInt8), - ParquetEncoding::RleDictionary, - &dict_raw, - &encoded, - idx.len(), - def_levels, - ) - .expect("decode_column failed"); - - match array { - Array::TextArray(TextArray::Categorical8(cat)) => { - assert_eq!(cat.data.as_slice(), &[0u8, 1, 1, 0]); - let uniq: Vec<_> = cat.unique_values.iter().collect(); - assert_eq!(uniq, vec!["foo", "bar"]); - } - _ => panic!("unexpected array variant {:?}", array), - } - } #[test] fn decode_column_plain_int32() { @@ -1381,8 +1643,8 @@ mod tests { let def_levels = vec![true; values.len()]; let array = decode_column( &ArrowType::Int32, - ParquetEncoding::Plain, - &[], + ParquetPhysicalType::Int32, + ValueLayout::Fixed(4), &buf, values.len(), def_levels.clone(), @@ -1401,13 +1663,14 @@ mod tests { #[test] fn decode_column_boolean_plain() { let bits = [true, false, true, true, false, false]; - let data_mask = Bitmask::from_bools(&bits); - let def_levels = bits.to_vec(); // no nulls + // one byte per row, as the page readers unpack boolean pages + let bytes: Vec = bits.iter().map(|&b| b as u8).collect(); + let def_levels = vec![true; bits.len()]; let array = decode_column( &ArrowType::Boolean, - ParquetEncoding::Plain, - &[], - data_mask.as_slice(), + ParquetPhysicalType::Boolean, + ValueLayout::Fixed(1), + &bytes, bits.len(), def_levels, ) diff --git a/rust/src/models/types/parquet.rs b/rust/src/models/types/parquet.rs index 424aa6a..8a932a9 100644 --- a/rust/src/models/types/parquet.rs +++ b/rust/src/models/types/parquet.rs @@ -13,7 +13,7 @@ use crate::error::IoError; #[cfg(feature = "datetime")] use minarrow::TimeUnit; -use minarrow::{ArrowType, ffi::arrow_dtype::CategoricalIndexType}; +use minarrow::ArrowType; /// Parquet physical types as defined in `parquet.thrift`. /// @@ -33,6 +33,8 @@ pub(crate) enum ParquetPhysicalType { Double = 5, /// Variable-length byte array (used for strings and binary data). ByteArray = 6, + /// Fixed-length byte array (used for Decimal128 and other fixed-width types). + FixedLenByteArray = 7, } impl ParquetPhysicalType { @@ -52,6 +54,7 @@ impl ParquetPhysicalType { 4 => Some(Self::Float), 5 => Some(Self::Double), 6 => Some(Self::ByteArray), + 7 => Some(Self::FixedLenByteArray), _ => None, } } @@ -67,21 +70,22 @@ pub(crate) enum ParquetLogicalType { NoneType, /// UTF-8 encoded string. Utf8, - /// 32-bit date - days since epoch + /// DATE - days since the Unix epoch, stored as INT32. #[cfg(feature = "datetime")] Date32, - /// 64-bit date - milliseconds since epoch + /// 64-bit timestamp - milliseconds since epoch. `utc` is Parquet's + /// `isAdjustedToUTC`: true for instants in UTC, false for local + /// wall-clock time with no zone. #[cfg(feature = "datetime")] - Date64, - /// 64-bit timestamp - milliseconds since epoch + TimestampMillis { utc: bool }, + /// 64-bit timestamp - microseconds since epoch, with the same `utc` + /// meaning as `TimestampMillis`. #[cfg(feature = "datetime")] - TimestampMillis, - /// 64-bit timestamp - microseconds since epoch + TimestampMicros { utc: bool }, + /// 64-bit timestamp - nanoseconds since epoch, with the same `utc` + /// meaning as `TimestampMillis`. Exists only as a `LogicalType`. #[cfg(feature = "datetime")] - TimestampMicros, - /// 64-bit timestamp - nanoseconds since epoch - #[cfg(feature = "datetime")] - TimestampNanos, + TimestampNanos { utc: bool }, /// 32-bit time - milliseconds since midnight #[cfg(feature = "datetime")] TimeMillis, @@ -136,10 +140,11 @@ impl ParquetLogicalType { Some(7) => Some(ParquetLogicalType::TimeMillis), #[cfg(feature = "datetime")] Some(8) => Some(ParquetLogicalType::TimeMicros), + // The legacy TIMESTAMP converted types are defined as UTC instants. #[cfg(feature = "datetime")] - Some(9) => Some(ParquetLogicalType::TimestampMillis), + Some(9) => Some(ParquetLogicalType::TimestampMillis { utc: true }), #[cfg(feature = "datetime")] - Some(10) => Some(ParquetLogicalType::TimestampMicros), + Some(10) => Some(ParquetLogicalType::TimestampMicros { utc: true }), Some(11) => Some(ParquetLogicalType::IntType { bit_width: 8, is_signed: false, @@ -295,17 +300,9 @@ pub(crate) fn arrow_type_to_parquet( is_signed: false, }, )), - #[cfg(any( - not(feature = "default_categorical_8"), - feature = "extended_categorical" - ))] - ArrowType::Dictionary(CategoricalIndexType::UInt32) => { - Ok((ParquetPhysicalType::ByteArray, ParquetLogicalType::Utf8)) - } - #[cfg(feature = "default_categorical_8")] - ArrowType::Dictionary(CategoricalIndexType::UInt8) => { - Ok((ParquetPhysicalType::ByteArray, ParquetLogicalType::Utf8)) - } + // Parquet has no dictionary type. A categorical column is a UTF8 + // string column whose pages the writer dictionary-encodes. + ArrowType::Dictionary(_) => Ok((ParquetPhysicalType::ByteArray, ParquetLogicalType::Utf8)), #[cfg(feature = "decimal")] ArrowType::Decimal32(p, s) => Ok(( ParquetPhysicalType::Int32, @@ -318,7 +315,7 @@ pub(crate) fn arrow_type_to_parquet( )), #[cfg(feature = "decimal")] ArrowType::Decimal128(p, s) => Ok(( - ParquetPhysicalType::Int64, + ParquetPhysicalType::FixedLenByteArray, ParquetLogicalType::Decimal { precision: *p, scale: *s }, )), ArrowType::Float32 => Ok((ParquetPhysicalType::Float, ParquetLogicalType::NoneType)), @@ -329,46 +326,39 @@ pub(crate) fn arrow_type_to_parquet( ArrowType::Utf8View => Ok((ParquetPhysicalType::ByteArray, ParquetLogicalType::Utf8)), #[cfg(feature = "datetime")] ArrowType::Date32 => Ok((ParquetPhysicalType::Int32, ParquetLogicalType::Date32)), + // Parquet DATE is a 32-bit day count, so Date64 milliseconds are + // carried into days on write. See `temporal_unit_scale`. #[cfg(feature = "datetime")] - ArrowType::Date64 => Ok((ParquetPhysicalType::Int64, ParquetLogicalType::Date64)), + ArrowType::Date64 => Ok((ParquetPhysicalType::Int32, ParquetLogicalType::Date32)), + // A timestamp with a timezone is an instant, so it is stored as + // adjusted to UTC. A timestamp without one is local wall-clock time. + // Parquet keeps only that flag, not the zone name. #[cfg(feature = "datetime")] - ArrowType::Timestamp(unit, _) => match unit { - TimeUnit::Milliseconds => Ok(( + ArrowType::Timestamp(unit, tz) => match unit { + // Parquet has no seconds unit, so seconds are scaled to + // milliseconds on write. + TimeUnit::Seconds | TimeUnit::Milliseconds => Ok(( ParquetPhysicalType::Int64, - ParquetLogicalType::TimestampMillis, + ParquetLogicalType::TimestampMillis { utc: tz.is_some() }, )), TimeUnit::Microseconds => Ok(( ParquetPhysicalType::Int64, - ParquetLogicalType::TimestampMicros, + ParquetLogicalType::TimestampMicros { utc: tz.is_some() }, )), TimeUnit::Nanoseconds => Ok(( ParquetPhysicalType::Int64, - ParquetLogicalType::TimestampNanos, + ParquetLogicalType::TimestampNanos { utc: tz.is_some() }, )), - TimeUnit::Seconds => Ok(( - ParquetPhysicalType::Int64, - ParquetLogicalType::TimestampMillis, - )), // best-effort - TimeUnit::Days => Ok((ParquetPhysicalType::Int64, ParquetLogicalType::Date64)), - }, - #[cfg(feature = "datetime")] - ArrowType::Time32(unit) => match unit { - TimeUnit::Milliseconds => { - Ok((ParquetPhysicalType::Int32, ParquetLogicalType::TimeMillis)) - } - TimeUnit::Microseconds => { - Ok((ParquetPhysicalType::Int32, ParquetLogicalType::TimeMicros)) - } - TimeUnit::Nanoseconds => { - Ok((ParquetPhysicalType::Int32, ParquetLogicalType::TimeNanos)) - } - TimeUnit::Seconds => Ok((ParquetPhysicalType::Int32, ParquetLogicalType::TimeMillis)), /* best-effort */ + // A timestamp counted in days is a date. TimeUnit::Days => Ok((ParquetPhysicalType::Int32, ParquetLogicalType::Date32)), }, + // Parquet fixes the storage width by unit: TIME(MILLIS) is INT32, + // TIME(MICROS) and TIME(NANOS) are INT64, whatever width the Arrow + // column uses. Seconds are scaled to milliseconds on write. #[cfg(feature = "datetime")] - ArrowType::Time64(unit) => match unit { - TimeUnit::Milliseconds => { - Ok((ParquetPhysicalType::Int64, ParquetLogicalType::TimeMillis)) + ArrowType::Time32(unit) | ArrowType::Time64(unit) => match unit { + TimeUnit::Seconds | TimeUnit::Milliseconds => { + Ok((ParquetPhysicalType::Int32, ParquetLogicalType::TimeMillis)) } TimeUnit::Microseconds => { Ok((ParquetPhysicalType::Int64, ParquetLogicalType::TimeMicros)) @@ -376,25 +366,39 @@ pub(crate) fn arrow_type_to_parquet( TimeUnit::Nanoseconds => { Ok((ParquetPhysicalType::Int64, ParquetLogicalType::TimeNanos)) } - TimeUnit::Seconds => Ok((ParquetPhysicalType::Int64, ParquetLogicalType::TimeMillis)), /* best-effort */ - TimeUnit::Days => Ok((ParquetPhysicalType::Int64, ParquetLogicalType::Date64)), + TimeUnit::Days => Err(IoError::UnsupportedType( + "time of day counted in days has no Parquet type".into(), + )), }, ArrowType::Null => Err(IoError::UnsupportedType( "Null type is not supported".into(), )), #[cfg(feature = "datetime")] - ArrowType::Duration32(_) => panic!("Duration does not map to a parquet type."), + ArrowType::Duration32(_) | ArrowType::Duration64(_) => Err(IoError::UnsupportedType( + "Duration has no Parquet logical type".into(), + )), #[cfg(feature = "datetime")] - ArrowType::Duration64(_) => panic!("Duration does not map to a parquet type."), + ArrowType::Interval(_) => Err(IoError::UnsupportedType( + "Interval is not supported".into(), + )), + } +} + +/// Multiplier and divisor that carry an Arrow temporal value into the unit +/// of its Parquet annotation from [`arrow_type_to_parquet`]. +/// +/// Most units match and scale by one. Seconds become milliseconds because +/// Parquet has no seconds unit, and Date64 milliseconds become the day +/// count that Parquet DATE stores. +pub(crate) fn temporal_unit_scale(ty: &ArrowType) -> (i64, i64) { + match ty { + #[cfg(feature = "datetime")] + ArrowType::Date64 => (1, 86_400_000), #[cfg(feature = "datetime")] - ArrowType::Interval(_) => panic!("Interval does not map to a parquet type."), - #[cfg(all(feature = "extended_categorical", feature = "extended_numeric_types"))] - &minarrow::ArrowType::Dictionary( - minarrow::ffi::arrow_dtype::CategoricalIndexType::UInt16, - ) - | &minarrow::ArrowType::Dictionary( - minarrow::ffi::arrow_dtype::CategoricalIndexType::UInt64, - ) => panic!(), + ArrowType::Timestamp(TimeUnit::Seconds, _) + | ArrowType::Time32(TimeUnit::Seconds) + | ArrowType::Time64(TimeUnit::Seconds) => (1000, 1), + _ => (1, 1), } } @@ -476,20 +480,20 @@ pub(crate) fn parquet_to_arrow_type( // Dates, times, timestamps #[cfg(feature = "datetime")] (ParquetPhysicalType::Int32, Some(ParquetLogicalType::Date32)) => Ok(ArrowType::Date32), + // A UTC-adjusted timestamp reads back with the "UTC" zone. Parquet + // records no zone name, so that is the only zone a file can carry. #[cfg(feature = "datetime")] - (ParquetPhysicalType::Int64, Some(ParquetLogicalType::Date64)) => Ok(ArrowType::Date64), - #[cfg(feature = "datetime")] - (ParquetPhysicalType::Int64, Some(ParquetLogicalType::TimestampMillis)) => { - Ok(ArrowType::Timestamp(TimeUnit::Milliseconds, None)) - } + (ParquetPhysicalType::Int64, Some(ParquetLogicalType::TimestampMillis { utc })) => Ok( + ArrowType::Timestamp(TimeUnit::Milliseconds, utc.then(|| "UTC".to_string())), + ), #[cfg(feature = "datetime")] - (ParquetPhysicalType::Int64, Some(ParquetLogicalType::TimestampMicros)) => { - Ok(ArrowType::Timestamp(TimeUnit::Microseconds, None)) - } + (ParquetPhysicalType::Int64, Some(ParquetLogicalType::TimestampMicros { utc })) => Ok( + ArrowType::Timestamp(TimeUnit::Microseconds, utc.then(|| "UTC".to_string())), + ), #[cfg(feature = "datetime")] - (ParquetPhysicalType::Int64, Some(ParquetLogicalType::TimestampNanos)) => { - Ok(ArrowType::Timestamp(TimeUnit::Nanoseconds, None)) - } + (ParquetPhysicalType::Int64, Some(ParquetLogicalType::TimestampNanos { utc })) => Ok( + ArrowType::Timestamp(TimeUnit::Nanoseconds, utc.then(|| "UTC".to_string())), + ), #[cfg(feature = "datetime")] (ParquetPhysicalType::Int32, Some(ParquetLogicalType::TimeMillis)) => { Ok(ArrowType::Time32(TimeUnit::Milliseconds)) @@ -515,17 +519,36 @@ pub(crate) fn parquet_to_arrow_type( Ok(ArrowType::Time64(TimeUnit::Nanoseconds)) } + // Decimals + #[cfg(feature = "decimal")] + ( + ParquetPhysicalType::Int32, + Some(ParquetLogicalType::Decimal { precision, scale }), + ) => Ok(ArrowType::Decimal32(precision, scale)), + #[cfg(feature = "decimal")] + ( + ParquetPhysicalType::Int64, + Some(ParquetLogicalType::Decimal { precision, scale }), + ) => Ok(ArrowType::Decimal64(precision, scale)), + // A fixed-length decimal may hold any precision. The narrowest + // Arrow decimal that fits the precision is used, matching the + // INT32 and INT64 forms above. + #[cfg(feature = "decimal")] + ( + ParquetPhysicalType::FixedLenByteArray, + Some(ParquetLogicalType::Decimal { precision, scale }), + ) => Ok(match precision { + 0..=9 => ArrowType::Decimal32(precision, scale), + 10..=18 => ArrowType::Decimal64(precision, scale), + _ => ArrowType::Decimal128(precision, scale), + }), + // Floats (ParquetPhysicalType::Float, _) => Ok(ArrowType::Float32), (ParquetPhysicalType::Double, _) => Ok(ArrowType::Float64), // Strings - always logical UTF8/Utf8 - #[cfg(not(feature = "large_string"))] (ParquetPhysicalType::ByteArray, Some(ParquetLogicalType::Utf8)) => Ok(ArrowType::String), - #[cfg(feature = "large_string")] - (ParquetPhysicalType::ByteArray, Some(ParquetLogicalType::Utf8)) => { - Ok(ArrowType::LargeString) - } // Fallback -- treat byte array without logical utf8 as unsupported (ParquetPhysicalType::ByteArray, None) => { @@ -567,9 +590,9 @@ mod tests { #[cfg(feature = "decimal")] #[test] - fn decimal128_maps_to_int64_physical() { + fn decimal128_maps_to_fixed_len_byte_array_physical() { let (phys, logical) = arrow_type_to_parquet(&ArrowType::Decimal128(38, 6)).unwrap(); - assert_eq!(phys, ParquetPhysicalType::Int64); + assert_eq!(phys, ParquetPhysicalType::FixedLenByteArray); assert_eq!( logical, ParquetLogicalType::Decimal { precision: 38, scale: 6 } diff --git a/rust/src/models/writers/parquet.rs b/rust/src/models/writers/parquet.rs index b12eb06..bbf5cd6 100644 --- a/rust/src/models/writers/parquet.rs +++ b/rust/src/models/writers/parquet.rs @@ -46,24 +46,28 @@ use std::io::{Seek, Write}; #[cfg(feature = "datetime")] use minarrow::TemporalArray; +#[cfg(feature = "datetime")] +use minarrow::ArrowType; use minarrow::{Array, NumericArray, Table, TextArray}; use crate::compression::{Compression, compress}; -use crate::constants::PARQUET_MAGIC; +use crate::constants::{PARQUET_CREATED_BY, PARQUET_MAGIC}; use crate::error::IoError; #[cfg(feature = "large_string")] use crate::models::encoders::parquet::data::encode_large_string_plain; use crate::models::encoders::parquet::data::{ encode_bool_bitpacked, encode_float32_plain, encode_float64_plain, encode_int32_plain, - encode_int64_plain, encode_string_plain, encode_uint32_as_int32_plain, - encode_uint64_as_int64_plain, + encode_int64_plain, encode_string_plain, encode_temporal_plain, + encode_uint32_as_int32_plain, encode_uint64_as_int64_plain, }; use crate::models::encoders::parquet::metadata::{ ColumnChunkMeta, ColumnMetadata, DataPageHeaderV2, DictionaryPageHeader, FileMetaData, PageHeader, PageType, RowGroupMeta, SchemaElement, Statistics, }; use crate::models::types::parquet::ParquetLogicalType::{self}; -use crate::models::types::parquet::{ParquetEncoding, arrow_type_to_parquet}; +use crate::models::types::parquet::{ + ParquetEncoding, ParquetPhysicalType, arrow_type_to_parquet, temporal_unit_scale, +}; // Chunk size for page splitting pub const PARQUET_PAGE_CHUNK_SIZE: usize = 32_768; @@ -110,17 +114,20 @@ pub fn write_parquet_table( scale: None, field_id: None, num_children: Some(table.cols.len() as i32), + logical_type: None, }); for (i, c) in table.cols.iter().enumerate() { let (physical, logical) = arrow_type_to_parquet(&c.field.dtype).unwrap(); + let (precision, scale, type_length) = decimal_schema_fields(&logical, physical); schema.push(SchemaElement { name: c.field.name.clone(), repetition_type: if c.field.nullable { 1 } else { 0 }, // OPTIONAL / REQUIRED type_: Some(physical), converted_type: logical_to_converted(&logical), - type_length: None, - precision: None, - scale: None, + logical_type: Some(logical), + type_length, + precision, + scale, field_id: Some(i as i32), num_children: None, }); @@ -136,6 +143,27 @@ pub fn write_parquet_table( // Column loop, multi-page support for col in &table.cols { + // Temporal columns are stored in the unit and width of their Parquet + // annotation, so their values may need scaling or narrowing on the + // way out. Every other type is stored as-is. + let (phys, _) = arrow_type_to_parquet(&col.field.dtype)?; + let temporal = match &col.field.dtype { + #[cfg(feature = "datetime")] + ArrowType::Date32 + | ArrowType::Date64 + | ArrowType::Timestamp(_, _) + | ArrowType::Time32(_) + | ArrowType::Time64(_) => true, + _ => false, + }; + let (unit_mul, unit_div) = temporal_unit_scale(&col.field.dtype); + let encode_temporal32 = |data: &[i32], out: &mut Vec| { + encode_temporal_plain(data, unit_mul, unit_div, phys, out) + }; + let encode_temporal64 = |data: &[i64], out: &mut Vec| { + encode_temporal_plain(data, unit_mul, unit_div, phys, out) + }; + let mut dictionary_page_offset = None; let mut encodings = vec![ParquetEncoding::Plain]; @@ -193,27 +221,82 @@ pub fn write_parquet_table( let end = usize::min(start + PARQUET_PAGE_CHUNK_SIZE, n); let len = end - start; - // encode the raw values for this slice + // rep / def levels for this chunk + let def_levels = col.array.null_mask().map_or_else( + || vec![true; len], + |mask| (start..end).map(|i| mask.get(i)).collect(), + ); + let has_nulls = def_levels.iter().any(|&v| !v); let mut values_raw = Vec::new(); + + // Encode a fixed-width slice for this page. The Parquet value + // section holds non-null values only, so null slots are dropped + // before encoding. + macro_rules! encode_valid { + ($encode:ident, $data:expr) => { + if has_nulls { + let valid: Vec<_> = $data + .iter() + .zip(&def_levels) + .filter(|(_, valid)| **valid) + .map(|(v, _)| *v) + .collect(); + $encode(&valid, &mut values_raw) + } else { + $encode($data, &mut values_raw) + } + }; + } + + // encode the raw values for this slice match &col.array { Array::NumericArray(n) => match n { + NumericArray::Int32(a) if temporal => { + encode_valid!(encode_temporal32, &a.data[start..end])? + } NumericArray::Int32(a) => { - encode_int32_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_int32_plain, &a.data[start..end]) } NumericArray::UInt32(a) => { - encode_uint32_as_int32_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_uint32_as_int32_plain, &a.data[start..end]) + } + NumericArray::Int64(a) if temporal => { + encode_valid!(encode_temporal64, &a.data[start..end])? } NumericArray::Int64(a) => { - encode_int64_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_int64_plain, &a.data[start..end]) } NumericArray::UInt64(a) => { - encode_uint64_as_int64_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_uint64_as_int64_plain, &a.data[start..end]) } NumericArray::Float32(a) => { - encode_float32_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_float32_plain, &a.data[start..end]) } NumericArray::Float64(a) => { - encode_float64_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_float64_plain, &a.data[start..end]) + } + #[cfg(feature = "decimal")] + NumericArray::Decimal32(a) => { + encode_valid!(encode_int32_plain, &a.data[start..end]) + } + #[cfg(feature = "decimal")] + NumericArray::Decimal64(a) => { + encode_valid!(encode_int64_plain, &a.data[start..end]) + } + #[cfg(feature = "decimal")] + NumericArray::Decimal128(a) => { + let slice = &a.data[start..end]; + if has_nulls { + for (v, valid) in slice.iter().zip(&def_levels) { + if *valid { + values_raw.extend_from_slice(&v.to_be_bytes()); + } + } + } else { + for v in slice { + values_raw.extend_from_slice(&v.to_be_bytes()); + } + } } _ => return Err(IoError::UnsupportedType("numeric".into())), }, @@ -251,33 +334,36 @@ pub fn write_parquet_table( )?, #[cfg(feature = "datetime")] Array::TemporalArray(TemporalArray::Datetime32(a)) => { - use crate::models::encoders::parquet::data::encode_datetime32_plain; - - encode_datetime32_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_temporal32, &a.data[start..end])? } #[cfg(feature = "datetime")] Array::TemporalArray(TemporalArray::Datetime64(a)) => { - use crate::models::encoders::parquet::data::encode_datetime64_plain; - - encode_datetime64_plain(&a.data[start..end], &mut values_raw) + encode_valid!(encode_temporal64, &a.data[start..end])? } #[cfg(any( not(feature = "default_categorical_8"), feature = "extended_categorical" ))] Array::TextArray(TextArray::Categorical32(a)) => { - encode_dictionary_indices_rle(&a.data[start..end], &mut values_raw)? + encode_valid!(encode_dictionary_indices_rle, &a.data[start..end])? } #[cfg(feature = "default_categorical_8")] Array::TextArray(TextArray::Categorical8(a)) => { - let idx: Vec = a.data[start..end].iter().map(|&v| v as u32).collect(); + let idx: Vec = a.data[start..end] + .iter() + .zip(&def_levels) + .filter(|(_, valid)| **valid) + .map(|(&v, _)| v as u32) + .collect(); encode_dictionary_indices_rle(&idx, &mut values_raw)? } #[cfg(all(feature = "extended_categorical", feature = "large_string"))] Array::TextArray(TextArray::Categorical64(a)) => { let idx: Vec = a.data[start..end] .iter() - .map(|&v| u32::try_from(v)) + .zip(&def_levels) + .filter(|(_, valid)| **valid) + .map(|(&v, _)| u32::try_from(v)) .collect::>() .map_err(|_| { IoError::Format( @@ -289,11 +375,6 @@ pub fn write_parquet_table( _ => return Err(IoError::UnsupportedType(format!("array {:?}", col.array))), } - // rep / def levels for this chunk - let def_levels = col.array.null_mask().map_or_else( - || vec![true; len], - |mask| (start..end).map(|i| mask.get(i)).collect(), - ); let def_buf = encode_levels_rle(&def_levels); let rep_buf = encode_levels_rle(&vec![false; len]); @@ -337,7 +418,8 @@ pub fn write_parquet_table( data_page_header_v2: Some(DataPageHeaderV2 { num_rows: len as i32, num_nulls: def_levels.iter().filter(|&&v| !v).count() as i32, - num_values: (len - def_levels.iter().filter(|&&v| !v).count()) as i32, + // num_values counts every row of the page, nulls included. + num_values: len as i32, encoding: if is_dictionary(&col.array) { ParquetEncoding::RleDictionary } else { @@ -369,7 +451,6 @@ pub fn write_parquet_table( // column-chunk metadata let first_data = recorded_data_page_offset.expect("at least one data page must be emitted"); - let (phys, _) = arrow_type_to_parquet(&col.field.dtype)?; columns_meta.push(ColumnChunkMeta { file_offset: first_data, meta_data: ColumnMetadata { @@ -389,7 +470,6 @@ pub fn write_parquet_table( data_page_offset: first_data, dictionary_page_offset, statistics: None, - definition_level: if col.field.nullable { 1 } else { 0 }, }, }); } @@ -408,7 +488,7 @@ pub fn write_parquet_table( num_rows: n_rows_i64, row_groups, key_value_metadata: None, - created_by: Some("parquet_writer-v2".into()), + created_by: Some(PARQUET_CREATED_BY.into()), } .write(&mut out)?; @@ -417,6 +497,25 @@ pub fn write_parquet_table( // Helpers +/// Extract precision, scale, and type_length for decimal schema elements. +/// Non-decimal types return `(None, None, None)`. +fn decimal_schema_fields( + logical: &ParquetLogicalType, + #[allow(unused_variables)] physical: ParquetPhysicalType, +) -> (Option, Option, Option) { + #[cfg(feature = "decimal")] + if let ParquetLogicalType::Decimal { precision, scale } = logical { + let type_length = if matches!(physical, ParquetPhysicalType::FixedLenByteArray) { + Some(16) + } else { + None + }; + return (Some(*precision as i32), Some(*scale as i32), type_length); + } + let _ = logical; + (None, None, None) +} + /// Add a dictionary page and return its uncompressed and compressed byte /// contributions to the column chunk totals, each including the page header. fn write_dictionary_page<'a, W, I>( @@ -560,12 +659,12 @@ fn logical_to_converted(log: &ParquetLogicalType) -> Option { ParquetLogicalType::Utf8 => 0, #[cfg(feature = "datetime")] ParquetLogicalType::Date32 => 6, + // The legacy TIMESTAMP converted types mean UTC instants, so a local + // timestamp carries only its LogicalType. #[cfg(feature = "datetime")] - ParquetLogicalType::Date64 => return None, - #[cfg(feature = "datetime")] - ParquetLogicalType::TimestampMillis => 9, + ParquetLogicalType::TimestampMillis { utc: true } => 9, #[cfg(feature = "datetime")] - ParquetLogicalType::TimestampMicros => 10, + ParquetLogicalType::TimestampMicros { utc: true } => 10, #[cfg(feature = "datetime")] ParquetLogicalType::TimeMillis => 7, #[cfg(feature = "datetime")] @@ -602,6 +701,8 @@ fn logical_to_converted(log: &ParquetLogicalType) -> Option { bit_width: 64, is_signed: true, } => 18, + #[cfg(feature = "decimal")] + ParquetLogicalType::Decimal { .. } => 5, _ => return None, }) } diff --git a/rust/tests/parquet_nullable_roundtrip.rs b/rust/tests/parquet_nullable_roundtrip.rs new file mode 100644 index 0000000..f1adff1 --- /dev/null +++ b/rust/tests/parquet_nullable_roundtrip.rs @@ -0,0 +1,573 @@ +// Copyright Peter G. Bower 2025-2026. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parquet write-then-read over every supported column type with nulls. +//! +//! Each column spans several data pages with nulls placed on both sides of +//! every page boundary, so the value sections hold non-null entries only +//! and the reader has to scatter them back against the definition levels +//! page by page. +//! +//! The types that come back are the ones the Parquet schema names, which +//! is not always the Arrow type that went in: categoricals are UTF8 +//! strings, Date64 is a DATE day count, seconds become milliseconds, and +//! TIME columns take the storage width their unit requires. + +#[cfg(feature = "parquet")] +mod parquet_nullable_roundtrip_tests { + use std::io::{Cursor, Seek, SeekFrom}; + + use lightstream::compression::Compression; + use lightstream::models::readers::parquet::load_parquet_table; + use lightstream::models::writers::parquet::{PARQUET_PAGE_CHUNK_SIZE, write_parquet_table}; + use minarrow::ffi::arrow_dtype::CategoricalIndexType; + use minarrow::{ + Array, ArrowType, Bitmask, BooleanArray, CategoricalArray, Field, FieldArray, FloatArray, + IntegerArray, MaskedArray, NumericArray, StringArray, Table, TextArray, Vec64, + }; + #[cfg(feature = "datetime")] + use minarrow::{DatetimeArray, TemporalArray, TimeUnit}; + #[cfg(feature = "decimal")] + use minarrow::DecimalArray; + + /// Rows per column: two full pages plus a partial third page. + const N_ROWS: usize = 2 * PARQUET_PAGE_CHUNK_SIZE + 13; + + /// Every seventh row from row three is null, plus the rows either side of + /// each page boundary. + fn is_valid(i: usize) -> bool { + let boundary = [ + PARQUET_PAGE_CHUNK_SIZE - 1, + PARQUET_PAGE_CHUNK_SIZE, + 2 * PARQUET_PAGE_CHUNK_SIZE - 1, + 2 * PARQUET_PAGE_CHUNK_SIZE, + ]; + i % 7 != 3 && !boundary.contains(&i) + } + + fn null_mask() -> Bitmask { + let bits: Vec = (0..N_ROWS).map(is_valid).collect(); + Bitmask::from_bools(&bits) + } + + fn expected(value: impl Fn(usize) -> T) -> Vec> { + (0..N_ROWS) + .map(|i| is_valid(i).then(|| value(i))) + .collect() + } + + fn int32_value(i: usize) -> i32 { + i as i32 - 1000 + } + fn uint32_value(i: usize) -> u32 { + i as u32 * 3 + } + fn int64_value(i: usize) -> i64 { + -(i as i64) * 7 + } + fn uint64_value(i: usize) -> u64 { + i as u64 + (1 << 40) + } + fn float32_value(i: usize) -> f32 { + i as f32 * 0.5 + } + fn float64_value(i: usize) -> f64 { + i as f64 * -0.25 + } + fn bool_value(i: usize) -> bool { + i % 3 == 0 + } + fn string_value(i: usize) -> String { + format!("s{}", i % 101) + } + fn category_value(i: usize) -> &'static str { + ["red", "green", "blue"][i % 3] + } + #[cfg(feature = "datetime")] + fn date32_value(i: usize) -> i32 { + i as i32 * 3 + } + /// Milliseconds with a sub-day remainder, so the DATE day count is the + /// truncated quotient. + #[cfg(feature = "datetime")] + fn date64_value(i: usize) -> i64 { + i as i64 * 86_400_000 + 12_345 + } + #[cfg(feature = "datetime")] + fn timestamp_value(i: usize) -> i64 { + 1_700_000_000_000 + i as i64 * 1_001 + } + #[cfg(feature = "datetime")] + fn time_value(i: usize) -> i64 { + (i % 86_400) as i64 * 7 + } + #[cfg(feature = "decimal")] + fn decimal32_value(i: usize) -> i32 { + i as i32 * 125 - 50_000 + } + #[cfg(feature = "decimal")] + fn decimal64_value(i: usize) -> i64 { + i as i64 * 1_000_003 - 7 + } + #[cfg(feature = "decimal")] + fn decimal128_value(i: usize) -> i128 { + (i as i128 - 5) * 1_000_000_000_000_000_000_000 + } + + fn column(name: &str, dtype: ArrowType, array: Array) -> FieldArray { + FieldArray::new(Field::new(name, dtype, true, None), array) + } + + #[cfg(feature = "datetime")] + fn temporal32(name: &str, dtype: ArrowType, unit: TimeUnit, value: fn(usize) -> i32) -> FieldArray { + column( + name, + dtype, + Array::from_datetime_i32(DatetimeArray::from_vec64( + (0..N_ROWS).map(value).collect::>(), + Some(null_mask()), + Some(unit), + )), + ) + } + + #[cfg(feature = "datetime")] + fn temporal64(name: &str, dtype: ArrowType, unit: TimeUnit, value: fn(usize) -> i64) -> FieldArray { + column( + name, + dtype, + Array::from_datetime_i64(DatetimeArray::from_vec64( + (0..N_ROWS).map(value).collect::>(), + Some(null_mask()), + Some(unit), + )), + ) + } + + fn all_types_table() -> Table { + let mask = null_mask(); + let strings: Vec = (0..N_ROWS).map(string_value).collect(); + let categories: Vec<&str> = (0..N_ROWS).map(category_value).collect(); + + let mut cols = vec![ + column( + "int32", + ArrowType::Int32, + Array::from_int32(IntegerArray::from_vec64( + (0..N_ROWS).map(int32_value).collect::>(), + Some(mask.clone()), + )), + ), + column( + "uint32", + ArrowType::UInt32, + Array::from_uint32(IntegerArray::from_vec64( + (0..N_ROWS).map(uint32_value).collect::>(), + Some(mask.clone()), + )), + ), + column( + "int64", + ArrowType::Int64, + Array::from_int64(IntegerArray::from_vec64( + (0..N_ROWS).map(int64_value).collect::>(), + Some(mask.clone()), + )), + ), + column( + "uint64", + ArrowType::UInt64, + Array::from_uint64(IntegerArray::from_vec64( + (0..N_ROWS).map(uint64_value).collect::>(), + Some(mask.clone()), + )), + ), + column( + "float32", + ArrowType::Float32, + Array::from_float32(FloatArray::from_vec64( + (0..N_ROWS).map(float32_value).collect::>(), + Some(mask.clone()), + )), + ), + column( + "float64", + ArrowType::Float64, + Array::from_float64(FloatArray::from_vec64( + (0..N_ROWS).map(float64_value).collect::>(), + Some(mask.clone()), + )), + ), + column( + "bool", + ArrowType::Boolean, + Array::from_bool(BooleanArray::new( + Bitmask::from_bools(&(0..N_ROWS).map(bool_value).collect::>()), + Some(mask.clone()), + )), + ), + column( + "string", + ArrowType::String, + Array::from_string32(StringArray::from_vec( + strings.iter().map(String::as_str).collect(), + Some(mask.clone()), + )), + ), + ]; + + #[cfg(feature = "default_categorical_8")] + cols.push(column( + "category", + ArrowType::Dictionary(CategoricalIndexType::UInt8), + Array::from_categorical8(CategoricalArray::::from_vec( + categories, + Some(mask.clone()), + )), + )); + #[cfg(not(feature = "default_categorical_8"))] + cols.push(column( + "category", + ArrowType::Dictionary(CategoricalIndexType::UInt32), + Array::from_categorical32(CategoricalArray::::from_vec( + categories, + Some(mask.clone()), + )), + )); + + #[cfg(feature = "datetime")] + { + cols.push(temporal32("date32", ArrowType::Date32, TimeUnit::Days, date32_value)); + cols.push(temporal64("date64", ArrowType::Date64, TimeUnit::Milliseconds, date64_value)); + cols.push(temporal64( + "ts_s", + ArrowType::Timestamp(TimeUnit::Seconds, None), + TimeUnit::Seconds, + timestamp_value, + )); + cols.push(temporal64( + "ts_ms", + ArrowType::Timestamp(TimeUnit::Milliseconds, None), + TimeUnit::Milliseconds, + timestamp_value, + )); + cols.push(temporal64( + "ts_us", + ArrowType::Timestamp(TimeUnit::Microseconds, None), + TimeUnit::Microseconds, + timestamp_value, + )); + cols.push(temporal64( + "ts_ns", + ArrowType::Timestamp(TimeUnit::Nanoseconds, None), + TimeUnit::Nanoseconds, + timestamp_value, + )); + cols.push(temporal64( + "ts_us_utc", + ArrowType::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string())), + TimeUnit::Microseconds, + timestamp_value, + )); + cols.push(temporal32( + "time32_ms", + ArrowType::Time32(TimeUnit::Milliseconds), + TimeUnit::Milliseconds, + |i| time_value(i) as i32, + )); + cols.push(temporal32( + "time32_us", + ArrowType::Time32(TimeUnit::Microseconds), + TimeUnit::Microseconds, + |i| time_value(i) as i32, + )); + cols.push(temporal64( + "time64_ms", + ArrowType::Time64(TimeUnit::Milliseconds), + TimeUnit::Milliseconds, + time_value, + )); + cols.push(temporal64( + "time64_ns", + ArrowType::Time64(TimeUnit::Nanoseconds), + TimeUnit::Nanoseconds, + time_value, + )); + } + + #[cfg(feature = "decimal")] + { + cols.push(column( + "dec32", + ArrowType::Decimal32(7, 2), + Array::from_decimal32(DecimalArray::from_vec64( + (0..N_ROWS).map(decimal32_value).collect::>(), + Some(mask.clone()), + 7, + 2, + )), + )); + cols.push(column( + "dec64", + ArrowType::Decimal64(18, 4), + Array::from_decimal64(DecimalArray::from_vec64( + (0..N_ROWS).map(decimal64_value).collect::>(), + Some(mask.clone()), + 18, + 4, + )), + )); + cols.push(column( + "dec128", + ArrowType::Decimal128(38, 6), + Array::from_decimal128(DecimalArray::from_vec64( + (0..N_ROWS).map(decimal128_value).collect::>(), + Some(mask.clone()), + 38, + 6, + )), + )); + } + + Table::new("all_types".to_string(), Some(cols)) + } + + fn roundtrip(table: &Table, compression: Option) -> Table { + let mut buf = Cursor::new(Vec::new()); + write_parquet_table(table, &mut buf, compression).expect("write"); + buf.seek(SeekFrom::Start(0)).unwrap(); + load_parquet_table(&mut buf).expect("read") + } + + fn col<'a>(table: &'a Table, name: &str) -> &'a FieldArray { + table + .cols + .iter() + .find(|c| c.field.name == name) + .unwrap_or_else(|| panic!("column {name} missing")) + } + + #[cfg(feature = "datetime")] + fn assert_temporal32(out: &Table, name: &str, dtype: ArrowType, unit: TimeUnit, value: impl Fn(usize) -> i32) { + assert_eq!(col(out, name).field.dtype, dtype, "{name} dtype"); + match &col(out, name).array { + Array::TemporalArray(TemporalArray::Datetime32(a)) => { + assert_eq!(a.time_unit, unit, "{name} unit"); + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(value), "{name}"); + } + other => panic!("{name}: {other:?}"), + } + } + + #[cfg(feature = "datetime")] + fn assert_temporal64(out: &Table, name: &str, dtype: ArrowType, unit: TimeUnit, value: impl Fn(usize) -> i64) { + assert_eq!(col(out, name).field.dtype, dtype, "{name} dtype"); + match &col(out, name).array { + Array::TemporalArray(TemporalArray::Datetime64(a)) => { + assert_eq!(a.time_unit, unit, "{name} unit"); + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(value), "{name}"); + } + other => panic!("{name}: {other:?}"), + } + } + + fn assert_all_types(out: &Table) { + let expected_nulls = (0..N_ROWS).filter(|&i| !is_valid(i)).count(); + assert_eq!(out.n_rows, N_ROWS); + for c in &out.cols { + assert!(c.field.nullable, "{} must be nullable", c.field.name); + assert_eq!(c.null_count, expected_nulls, "{} null count", c.field.name); + assert_eq!(c.array.len(), N_ROWS, "{} length", c.field.name); + } + + match &col(out, "int32").array { + Array::NumericArray(NumericArray::Int32(a)) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(int32_value)) + } + other => panic!("int32: {other:?}"), + } + match &col(out, "uint32").array { + Array::NumericArray(NumericArray::UInt32(a)) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(uint32_value)) + } + other => panic!("uint32: {other:?}"), + } + match &col(out, "int64").array { + Array::NumericArray(NumericArray::Int64(a)) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(int64_value)) + } + other => panic!("int64: {other:?}"), + } + match &col(out, "uint64").array { + Array::NumericArray(NumericArray::UInt64(a)) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(uint64_value)) + } + other => panic!("uint64: {other:?}"), + } + match &col(out, "float32").array { + Array::NumericArray(NumericArray::Float32(a)) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(float32_value)) + } + other => panic!("float32: {other:?}"), + } + match &col(out, "float64").array { + Array::NumericArray(NumericArray::Float64(a)) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(float64_value)) + } + other => panic!("float64: {other:?}"), + } + match &col(out, "bool").array { + Array::BooleanArray(a) => { + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(bool_value)) + } + other => panic!("bool: {other:?}"), + } + assert_eq!(col(out, "string").field.dtype, ArrowType::String); + match &col(out, "string").array { + Array::TextArray(TextArray::String32(a)) => assert_eq!( + (0..N_ROWS).map(|i| a.get(i).map(str::to_owned)).collect::>(), + expected(string_value) + ), + other => panic!("string: {other:?}"), + } + + // A categorical column is a UTF8 string column in Parquet, so it + // reads back as one. + assert_eq!(col(out, "category").field.dtype, ArrowType::String); + match &col(out, "category").array { + Array::TextArray(TextArray::String32(a)) => assert_eq!( + (0..N_ROWS).map(|i| a.get(i).map(str::to_owned)).collect::>(), + expected(|i| category_value(i).to_owned()) + ), + other => panic!("category: {other:?}"), + } + + #[cfg(feature = "datetime")] + { + assert_temporal32(out, "date32", ArrowType::Date32, TimeUnit::Days, date32_value); + // DATE stores days, so Date64 milliseconds come back as Date32. + assert_temporal32(out, "date64", ArrowType::Date32, TimeUnit::Days, |i| { + (date64_value(i) / 86_400_000) as i32 + }); + // Parquet has no seconds unit, so seconds come back as milliseconds. + assert_temporal64( + out, + "ts_s", + ArrowType::Timestamp(TimeUnit::Milliseconds, None), + TimeUnit::Milliseconds, + |i| timestamp_value(i) * 1000, + ); + assert_temporal64( + out, + "ts_ms", + ArrowType::Timestamp(TimeUnit::Milliseconds, None), + TimeUnit::Milliseconds, + timestamp_value, + ); + assert_temporal64( + out, + "ts_us", + ArrowType::Timestamp(TimeUnit::Microseconds, None), + TimeUnit::Microseconds, + timestamp_value, + ); + assert_temporal64( + out, + "ts_ns", + ArrowType::Timestamp(TimeUnit::Nanoseconds, None), + TimeUnit::Nanoseconds, + timestamp_value, + ); + // A zoned timestamp is stored as adjusted to UTC and reads back + // with the UTC zone. + assert_temporal64( + out, + "ts_us_utc", + ArrowType::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string())), + TimeUnit::Microseconds, + timestamp_value, + ); + assert_temporal32( + out, + "time32_ms", + ArrowType::Time32(TimeUnit::Milliseconds), + TimeUnit::Milliseconds, + |i| time_value(i) as i32, + ); + // TIME(MICROS) is INT64 in Parquet, so Time32 microseconds widen. + assert_temporal64( + out, + "time32_us", + ArrowType::Time64(TimeUnit::Microseconds), + TimeUnit::Microseconds, + time_value, + ); + // TIME(MILLIS) is INT32 in Parquet, so Time64 milliseconds narrow. + assert_temporal32( + out, + "time64_ms", + ArrowType::Time32(TimeUnit::Milliseconds), + TimeUnit::Milliseconds, + |i| time_value(i) as i32, + ); + assert_temporal64( + out, + "time64_ns", + ArrowType::Time64(TimeUnit::Nanoseconds), + TimeUnit::Nanoseconds, + time_value, + ); + } + + #[cfg(feature = "decimal")] + { + assert_eq!(col(out, "dec32").field.dtype, ArrowType::Decimal32(7, 2)); + match &col(out, "dec32").array { + Array::NumericArray(NumericArray::Decimal32(a)) => { + assert_eq!((a.precision, a.scale), (7, 2)); + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(decimal32_value)); + } + other => panic!("dec32: {other:?}"), + } + assert_eq!(col(out, "dec64").field.dtype, ArrowType::Decimal64(18, 4)); + match &col(out, "dec64").array { + Array::NumericArray(NumericArray::Decimal64(a)) => { + assert_eq!((a.precision, a.scale), (18, 4)); + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(decimal64_value)); + } + other => panic!("dec64: {other:?}"), + } + assert_eq!(col(out, "dec128").field.dtype, ArrowType::Decimal128(38, 6)); + match &col(out, "dec128").array { + Array::NumericArray(NumericArray::Decimal128(a)) => { + assert_eq!((a.precision, a.scale), (38, 6)); + assert_eq!((0..N_ROWS).map(|i| a.get(i)).collect::>(), expected(decimal128_value)); + } + other => panic!("dec128: {other:?}"), + } + } + } + + #[test] + fn nullable_all_types_uncompressed() { + let table = all_types_table(); + assert_all_types(&roundtrip(&table, None)); + } + + #[cfg(feature = "snappy")] + #[test] + fn nullable_all_types_snappy() { + let table = all_types_table(); + assert_all_types(&roundtrip(&table, Some(Compression::Snappy))); + } + + #[cfg(feature = "zstd")] + #[test] + fn nullable_all_types_zstd() { + let table = all_types_table(); + assert_all_types(&roundtrip(&table, Some(Compression::Zstd))); + } +} diff --git a/rust/tests/pyarrow_parquet.rs b/rust/tests/pyarrow_parquet.rs new file mode 100644 index 0000000..4401acd --- /dev/null +++ b/rust/tests/pyarrow_parquet.rs @@ -0,0 +1,333 @@ +// Copyright Peter G. Bower 2025-2026. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parquet reader conformance against files written by pyarrow. +//! +//! The fixtures under `pyarrow-roundtrip/` are committed and regenerated +//! with `generate_pyarrow_parquet_files.py`. They cover the page layouts +//! parquet-cpp writes by default (Snappy, dictionary encoding, DataPageV1, +//! multiple row groups) plus the DataPageV2 layouts with and without +//! dictionaries. + +#[cfg(feature = "parquet")] +mod pyarrow_parquet_tests { + use std::fs::File; + use std::io::BufReader; + + use lightstream::models::readers::parquet::{load_parquet_table, load_parquet_table_cols}; + use minarrow::{Array, ArrowType, MaskedArray, NumericArray, Table, TextArray}; + #[cfg(all(feature = "datetime", feature = "decimal", feature = "snappy"))] + use minarrow::{TemporalArray, TimeUnit}; + + fn open(name: &str) -> BufReader { + let path = format!("{}/pyarrow-roundtrip/{name}", env!("CARGO_MANIFEST_DIR")); + BufReader::new(File::open(&path).unwrap_or_else(|e| panic!("open {path}: {e}"))) + } + + fn column_dtype(table: &Table, name: &str) -> ArrowType { + table + .cols + .iter() + .find(|c| c.field.name == name) + .unwrap_or_else(|| panic!("column {name} missing")) + .field + .dtype + .clone() + } + + fn column<'a>(table: &'a Table, name: &str) -> &'a Array { + let col = table + .cols + .iter() + .find(|c| c.field.name == name) + .unwrap_or_else(|| panic!("column {name} missing")); + &col.array + } + + fn int32_values(table: &Table, name: &str) -> Vec> { + match column(table, name) { + Array::NumericArray(NumericArray::Int32(a)) => (0..a.len()).map(|i| a.get(i)).collect(), + other => panic!("{name}: expected Int32, got {other:?}"), + } + } + + fn int64_values(table: &Table, name: &str) -> Vec> { + match column(table, name) { + Array::NumericArray(NumericArray::Int64(a)) => (0..a.len()).map(|i| a.get(i)).collect(), + other => panic!("{name}: expected Int64, got {other:?}"), + } + } + + fn float32_values(table: &Table, name: &str) -> Vec> { + match column(table, name) { + Array::NumericArray(NumericArray::Float32(a)) => { + (0..a.len()).map(|i| a.get(i)).collect() + } + other => panic!("{name}: expected Float32, got {other:?}"), + } + } + + fn float64_values(table: &Table, name: &str) -> Vec> { + match column(table, name) { + Array::NumericArray(NumericArray::Float64(a)) => { + (0..a.len()).map(|i| a.get(i)).collect() + } + other => panic!("{name}: expected Float64, got {other:?}"), + } + } + + fn bool_values(table: &Table, name: &str) -> Vec> { + match column(table, name) { + Array::BooleanArray(a) => (0..a.len()).map(|i| a.get(i)).collect(), + other => panic!("{name}: expected Boolean, got {other:?}"), + } + } + + fn string_values(table: &Table, name: &str) -> Vec> { + match column(table, name) { + Array::TextArray(TextArray::String32(a)) => { + (0..a.len()).map(|i| a.get(i).map(str::to_owned)).collect() + } + other => panic!("{name}: expected String, got {other:?}"), + } + } + + /// Assert the seven-row table written by `nullable_table()` in the + /// generator script, in schema order. + fn assert_nullable_table(table: &Table) { + assert_eq!(table.n_rows, 7); + assert_eq!(table.cols.len(), 6); + let names: Vec<&str> = table.cols.iter().map(|c| c.field.name.as_str()).collect(); + assert_eq!( + names, + ["int32", "int64", "float32", "float64", "bool", "string"] + ); + assert!(table.cols.iter().all(|c| c.field.nullable)); + + assert_eq!( + int32_values(table, "int32"), + [Some(1), None, Some(3), Some(4), None, Some(6), Some(7)] + ); + assert_eq!( + int64_values(table, "int64"), + [Some(100), Some(101), None, Some(103), Some(104), Some(105), None] + ); + assert_eq!( + float32_values(table, "float32"), + [Some(0.5), Some(1.5), Some(2.5), None, Some(4.5), Some(5.5), Some(6.5)] + ); + assert_eq!( + float64_values(table, "float64"), + [None, Some(-1.0), Some(-2.0), Some(-3.0), Some(-4.0), None, Some(-6.0)] + ); + assert_eq!( + bool_values(table, "bool"), + [Some(true), Some(false), None, Some(true), Some(false), Some(true), None] + ); + assert_eq!( + string_values(table, "string"), + [ + Some("a".into()), + Some("b".into()), + Some("a".into()), + None, + Some("c".into()), + Some("b".into()), + Some("a".into()) + ] + ); + for col in &table.cols { + assert_eq!(col.null_count, col.array.null_count(), "{}", col.field.name); + } + } + + /// The three-column file from the original defect report, written + /// with pyarrow's defaults: Snappy, dictionary encoding, DataPageV1 + /// and REQUIRED columns. + #[cfg(feature = "snappy")] + #[test] + fn reads_pyarrow_default_layout() { + let table = load_parquet_table(open("pyarrow_simple.parquet")).expect("read"); + assert_eq!(table.n_rows, 5); + assert_eq!(table.cols.len(), 3); + assert_eq!(table.cols[0].field.dtype, ArrowType::Int64); + assert_eq!(table.cols[1].field.dtype, ArrowType::String); + assert_eq!(table.cols[2].field.dtype, ArrowType::Float64); + + assert_eq!( + int64_values(&table, "id"), + [Some(1), Some(2), Some(3), Some(4), Some(5)] + ); + assert_eq!( + string_values(&table, "name"), + ["Alice", "Bob", "Charlie", "Diana", "Eve"] + .map(|s| Some(s.to_owned())) + ); + assert_eq!( + float64_values(&table, "score"), + [Some(85.5), Some(92.0), Some(78.3), Some(95.1), Some(88.7)] + ); + assert!(table.cols.iter().all(|c| c.null_count == 0)); + } + + /// Nullable columns holding nulls, split over three row groups, with + /// pyarrow's default page layout. + #[cfg(feature = "snappy")] + #[test] + fn reads_pyarrow_nullable_columns_across_row_groups() { + let table = + load_parquet_table(open("pyarrow_nullable_row_groups.parquet")).expect("read"); + assert_nullable_table(&table); + } + + /// DataPageV2 pages with PLAIN values and no compression. + #[test] + fn reads_pyarrow_plain_data_page_v2() { + let table = load_parquet_table(open("pyarrow_plain_v2.parquet")).expect("read"); + assert_nullable_table(&table); + } + + /// DataPageV2 pages with dictionary-encoded values and Snappy + /// compression on the value section only. + #[cfg(feature = "snappy")] + #[test] + fn reads_pyarrow_dictionary_data_page_v2() { + let table = load_parquet_table(open("pyarrow_dictionary_v2.parquet")).expect("read"); + assert_nullable_table(&table); + } + + /// Temporal and decimal columns written with pyarrow's defaults. The + /// nanosecond timestamp exists only through the `LogicalType` + /// annotation, and pyarrow stores every decimal as a + /// FIXED_LEN_BYTE_ARRAY sized to its precision. + #[cfg(all(feature = "datetime", feature = "decimal", feature = "snappy"))] + #[test] + fn reads_pyarrow_temporal_and_decimal_columns() { + let table = load_parquet_table(open("pyarrow_temporal_decimal.parquet")).expect("read"); + assert_eq!(table.n_rows, 5); + + fn temporal32(table: &Table, name: &str, unit: TimeUnit) -> Vec> { + match column(table, name) { + Array::TemporalArray(TemporalArray::Datetime32(a)) => { + assert_eq!(a.time_unit, unit, "{name} unit"); + (0..a.len()).map(|i| a.get(i)).collect() + } + other => panic!("{name}: expected Datetime32, got {other:?}"), + } + } + fn temporal64(table: &Table, name: &str, unit: TimeUnit) -> Vec> { + match column(table, name) { + Array::TemporalArray(TemporalArray::Datetime64(a)) => { + assert_eq!(a.time_unit, unit, "{name} unit"); + (0..a.len()).map(|i| a.get(i)).collect() + } + other => panic!("{name}: expected Datetime64, got {other:?}"), + } + } + + assert_eq!(column_dtype(&table, "date"), ArrowType::Date32); + assert_eq!( + temporal32(&table, "date", TimeUnit::Days), + [Some(0), Some(1), None, Some(19_000), Some(-5)] + ); + assert_eq!(column_dtype(&table, "time_ms"), ArrowType::Time32(TimeUnit::Milliseconds)); + assert_eq!( + temporal32(&table, "time_ms", TimeUnit::Milliseconds), + [Some(0), Some(1_000), None, Some(43_200_000), Some(86_399_999)] + ); + assert_eq!(column_dtype(&table, "time_us"), ArrowType::Time64(TimeUnit::Microseconds)); + assert_eq!( + temporal64(&table, "time_us", TimeUnit::Microseconds), + [Some(0), None, Some(2_000_000), Some(43_200_000_000), Some(86_399_999_999)] + ); + assert_eq!( + column_dtype(&table, "ts_ms"), + ArrowType::Timestamp(TimeUnit::Milliseconds, None) + ); + assert_eq!( + temporal64(&table, "ts_ms", TimeUnit::Milliseconds), + [Some(0), Some(1_700_000_000_000), None, Some(-1), Some(86_400_000)] + ); + assert_eq!( + column_dtype(&table, "ts_us"), + ArrowType::Timestamp(TimeUnit::Microseconds, None) + ); + assert_eq!( + temporal64(&table, "ts_us", TimeUnit::Microseconds), + [Some(0), Some(1_700_000_000_000_000), None, Some(-1), Some(1)] + ); + assert_eq!( + column_dtype(&table, "ts_ns"), + ArrowType::Timestamp(TimeUnit::Nanoseconds, None) + ); + assert_eq!( + temporal64(&table, "ts_ns", TimeUnit::Nanoseconds), + [Some(0), Some(1_700_000_000_000_000_000), None, Some(-1), Some(1)] + ); + // pyarrow writes a zoned timestamp with isAdjustedToUTC set. + assert_eq!( + column_dtype(&table, "ts_us_utc"), + ArrowType::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string())) + ); + assert_eq!( + temporal64(&table, "ts_us_utc", TimeUnit::Microseconds), + [Some(0), Some(1_700_000_000_000_000), None, Some(-1), Some(1)] + ); + + assert_eq!(column_dtype(&table, "dec32"), ArrowType::Decimal32(7, 2)); + match column(&table, "dec32") { + Array::NumericArray(NumericArray::Decimal32(a)) => assert_eq!( + (0..a.len()).map(|i| a.get(i)).collect::>(), + [Some(125), None, Some(-350), Some(1), Some(9_999_999)] + ), + other => panic!("dec32: {other:?}"), + } + assert_eq!(column_dtype(&table, "dec64"), ArrowType::Decimal64(18, 4)); + match column(&table, "dec64") { + Array::NumericArray(NumericArray::Decimal64(a)) => assert_eq!( + (0..a.len()).map(|i| a.get(i)).collect::>(), + [Some(12_345), Some(-10_000), None, Some(1), Some(123_456_789_012_345_678)] + ), + other => panic!("dec64: {other:?}"), + } + assert_eq!(column_dtype(&table, "dec128"), ArrowType::Decimal128(38, 6)); + match column(&table, "dec128") { + Array::NumericArray(NumericArray::Decimal128(a)) => assert_eq!( + (0..a.len()).map(|i| a.get(i)).collect::>(), + [ + Some(1_000_001), + None, + Some(-1_000_001), + Some(0), + Some(12_345_678_901_234_567_890_123_456_789_012_123_456i128) + ] + ), + other => panic!("dec128: {other:?}"), + } + } + + /// Column projection over a pyarrow file selects by name and keeps + /// the values of the selected columns intact. + #[cfg(feature = "snappy")] + #[test] + fn projects_columns_from_pyarrow_file() { + let table = + load_parquet_table_cols(open("pyarrow_simple.parquet"), &["score", "name"]) + .expect("read"); + assert_eq!(table.n_rows, 5); + let names: Vec<&str> = table.cols.iter().map(|c| c.field.name.as_str()).collect(); + assert_eq!(names, ["name", "score"]); + assert_eq!( + string_values(&table, "name"), + ["Alice", "Bob", "Charlie", "Diana", "Eve"] + .map(|s| Some(s.to_owned())) + ); + assert_eq!( + float64_values(&table, "score"), + [Some(85.5), Some(92.0), Some(78.3), Some(95.1), Some(88.7)] + ); + } +}