Add decimal IPC parquet support

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 Notable changes are recorded from 0.5.0 onward.
 
+## 0.7.0
+
+### Changed
+
+- Minarrow upgraded to 0.18.1 and vec64 0.5.1.
+- New `decimal` feature forwarding minarrow's `decimal` feature. Decimal32, Decimal64 and Decimal128 columns are supported in Arrow IPC and in Parquet, where they map to the DECIMAL logical type over INT32, INT64 and FIXED_LEN_BYTE_ARRAY.
+- The Python package pins minarrow and minarrow-pyo3 at 0.18.1.
+- Parquet writer improvements:
+  - The Parquet writer follows the Parquet value layout for nullable columns.
+  - Several improvements in categorical, time, and date roundtripping.
+  - Files with nulls written by earlier releases do not read back under this release.
+  - Replaced packing for Duration and Interval columns with `UnsupportedType`
+
+### Fixed
+
+- The Parquet reader failed with `UnexpectedEof` on files written by pyarrow. It now reads DataPageV1 pages with compressed levels, `RLE` booleans, dictionary-encoded columns of any physical type, and files with several row groups.
+
+## 0.6.1
+
+### Changed
+
+- minarrow version bump to 0.17.0, vec64 0.5.0, arrow 59.2.0 and polars 0.55.2.
+
 ## 0.6.0
 
 ### Changed

python/Cargo.toml

@@ -2,7 +2,7 @@ cargo-features = ["trim-paths"]
 
 [package]
 name = "lightstream-py"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2024"
 authors = ["Peter G. Bower"]
 license = "MPL-2.0"
@@ -19,13 +19,13 @@ name = "lightstream_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-lightstream = { version = "0.6", path = "../rust", features = ["csv", "datetime", "extended_categorical", "extended_numeric_types", "json", "mmap", "http", "parquet", "protocol", "quic", "snappy", "stdio", "tcp", "tls", "uds", "webtransport", "websocket", "zstd"] }
+lightstream = { version = "0.7", path = "../rust", features = ["csv", "datetime", "decimal", "extended_categorical", "extended_numeric_types", "json", "mmap", "http", "parquet", "protocol", "quic", "snappy", "stdio", "tcp", "tls", "uds", "webtransport", "websocket", "zstd"] }
 # The categorical and numeric feature set mirrors the minarrow-py build.
 # minarrow-pyo3's dictionary-index conversion needs the extended features,
 # and they flow through lightstream's flags so its match arms gate in step
 # with minarrow's variants.
-minarrow = { version = "0.16", features = ["chunked"] }
-minarrow-pyo3 = { version = "0.16", features = ["extended_categorical", "extended_numeric_types"] }
+minarrow = { version = "0.18.1", features = ["chunked"] }
+minarrow-pyo3 = { version = "0.18.1", features = ["extended_categorical", "extended_numeric_types"] }
 futures-core = "0.3"
 pyo3 = { version = "0.29", features = ["abi3-py39"] }
 # The QUIC and WebTransport dependency pins mirror lightstream's, so

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "lightstream-io"
-version = "0.6.0"
+version = "0.7.0"
 description = "Streaming Arrow I/O for Python - files, sockets, and network transports with zero-copy minarrow interop."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -20,7 +20,7 @@ classifiers = [
     "Topic :: Scientific/Engineering",
     "Topic :: Software Development :: Libraries",
 ]
-dependencies = ["minarrow>=0.16"]
+dependencies = ["minarrow>=0.18.1"]
 
 [project.urls]
 Homepage = "https://github.com/SpaceCell/lightstream"

python/tests/test_files.py

@@ -11,10 +11,12 @@
 """
 
 import gc
+from decimal import Decimal
 
 import lightstream as ls
 import minarrow
 import pyarrow as pa
+import pyarrow.parquet as pq
 import pytest
 
 
@@ -224,6 +226,44 @@ def test_parquet_multi_write_consolidates(tmp_path):
     assert result.num_rows == 6
 
 
+def nullable_table():
+    return pa.table(
+        {
+            "id": pa.array([1, None, 3, 4, None], type=pa.int64()),
+            "small": pa.array([None, -2, 3, -4, 5], type=pa.int32()),
+            "count": pa.array([1, 2, None, 4, 5], type=pa.uint32()),
+            "name": pa.array(["a", "b", None, "a", "c"], type=pa.string()),
+            "score": pa.array([1.5, None, 3.5, 4.5, 5.5], type=pa.float64()),
+            "ratio": pa.array([0.5, 1.5, 2.5, None, 4.5], type=pa.float32()),
+            "flag": pa.array([True, False, None, True, False], type=pa.bool_()),
+            "day": pa.array([1, None, 3, 4, 5], type=pa.date32()),
+            "at": pa.array([1_000, 2_000, None, 4_000, 5_000], type=pa.timestamp("ns")),
+            "amount": pa.array(
+                [Decimal("1.25"), None, Decimal("-3.50"), Decimal("0.01"), Decimal("99.99")],
+                type=pa.decimal128(10, 2),
+            ),
+        }
+    )
+
+
+def test_parquet_reads_pyarrow_file_with_nulls(tmp_path):
+    path = str(tmp_path / "pyarrow.parquet")
+    original = nullable_table()
+    pq.write_table(original, path)
+
+    result = pa.table(ls.read(path).read_all())
+    assert result.to_pydict() == original.to_pydict()
+
+
+def test_parquet_written_with_nulls_reads_in_pyarrow(tmp_path):
+    path = str(tmp_path / "lightstream.parquet")
+    original = nullable_table()
+    with ls.write(path) as w:
+        w.write(original)
+
+    assert pq.read_table(path).to_pydict() == original.to_pydict()
+
+
 @pytest.mark.parametrize("codec", ["zstd", "snappy"])
 def test_parquet_compression(tmp_path, codec):
     path = str(tmp_path / f"quotes_{codec}.parquet")