From 6b6341b24c35d6870e3884d80105644f55abeaec Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 11:01:01 +0200 Subject: [PATCH 01/19] Add lazy remote Parquet table access --- ...emote_parquet_chicago_http_optimized.jsonl | 5 + .../remote_parquet_chicago_http_results.json | 61 ++ bench/remote_parquet_chicago_results.json | 61 ++ bench/remote_parquet_files.py | 194 +++++ bench/remote_parquet_synthetic_optimized.txt | 5 + bench/remote_parquet_traffic.py | 62 ++ doc/guides/remote_tables.md | 52 ++ plans/remote-parquet.md | 541 ++++++++++++ src/blosc2/b2objects.py | 4 + src/blosc2/ctable_remote_read.py | 22 + src/blosc2/remote_ctable.py | 115 ++- src/blosc2/remote_parquet.py | 822 ++++++++++++++++++ src/blosc2/schunk.py | 43 +- tests/ctable/test_remote_parquet.py | 649 ++++++++++++++ 14 files changed, 2632 insertions(+), 4 deletions(-) create mode 100644 bench/remote_parquet_chicago_http_optimized.jsonl create mode 100644 bench/remote_parquet_chicago_http_results.json create mode 100644 bench/remote_parquet_chicago_results.json create mode 100644 bench/remote_parquet_files.py create mode 100644 bench/remote_parquet_synthetic_optimized.txt create mode 100644 bench/remote_parquet_traffic.py create mode 100644 plans/remote-parquet.md create mode 100644 src/blosc2/remote_parquet.py create mode 100644 tests/ctable/test_remote_parquet.py diff --git a/bench/remote_parquet_chicago_http_optimized.jsonl b/bench/remote_parquet_chicago_http_optimized.jsonl new file mode 100644 index 000000000..21df62ebc --- /dev/null +++ b/bench/remote_parquet_chicago_http_optimized.jsonl @@ -0,0 +1,5 @@ +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.188, "cold_reads": 1, "cold_bytes": 331590, "cold_heads": 0, "cold_seconds": 17.571, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 34695136, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32.parquet", "file_bytes": 686857703, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.022, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 17.638, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 34695136, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f64.parquet", "file_bytes": 687450036, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.016, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 17.862, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.069, "cache_bytes": 34695136, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi.parquet", "file_bytes": 640173859, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 26, "open_bytes": 333139, "open_heads": 1, "open_seconds": 0.364, "cold_reads": 1, "cold_bytes": 6660483, "cold_heads": 0, "cold_seconds": 1.083, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 12632079, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "company", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.189, "cold_reads": 1, "cold_bytes": 1831, "cold_heads": 0, "cold_seconds": 0.819, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 34695431, "warm_equal": true, "groups": 25, "arrow_equal": true} diff --git a/bench/remote_parquet_chicago_http_results.json b/bench/remote_parquet_chicago_http_results.json new file mode 100644 index 000000000..07c4dec62 --- /dev/null +++ b/bench/remote_parquet_chicago_http_results.json @@ -0,0 +1,61 @@ +{ + "date": "2026-09-25", + "method": "Each file served by bench/remote_parquet_files.py --http on 127.0.0.1 with byte-range responses. Reads and bytes count completed server GET responses and response bodies; HEAD requests are counted separately. Headers and TCP overhead are excluded. Default in-memory converted cache; fsspec block_size=4096, cache_type=none. Cold access reads the final row; warm access repeats it in the same process.", + "files": [ + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "trip.sec", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 32679571, "seconds": 1.253}, + "cold": {"gets": 1, "heads": 0, "bytes": 331590, "seconds": 17.688}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32.parquet", + "column": "trip.sec", + "file_bytes": 686857703, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 33057447, "seconds": 1.084}, + "cold": {"gets": 1, "heads": 0, "bytes": 352939, "seconds": 17.885}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f64.parquet", + "column": "trip.sec", + "file_bytes": 687450036, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 33068884, "seconds": 1.122}, + "cold": {"gets": 1, "heads": 0, "bytes": 352939, "seconds": 18.131}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "company", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 32679571, "seconds": 1.308}, + "cold": {"gets": 1, "heads": 0, "bytes": 1831, "seconds": 0.813}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.068}, + "arrow_equal": true + }, + { + "file": "chicago-taxi.parquet", + "column": "trip.sec", + "file_bytes": 640173859, + "rows": 24314322, + "groups": 25, + "open": {"gets": 27, "heads": 2, "bytes": 671548047, "seconds": 3.909}, + "cold": {"gets": 1, "heads": 0, "bytes": 6660483, "seconds": 1.051}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.067}, + "arrow_equal": true + } + ] +} diff --git a/bench/remote_parquet_chicago_results.json b/bench/remote_parquet_chicago_results.json new file mode 100644 index 000000000..8908c637f --- /dev/null +++ b/bench/remote_parquet_chicago_results.json @@ -0,0 +1,61 @@ +{ + "date": "2026-09-25", + "method": "Local file-handle read calls and returned bytes from bench/remote_parquet_files.py; these are not HTTP or S3 wire requests. Cold access reads the final row; warm access repeats it in the same process. Default memory cache; no disk cache.", + "files": [ + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "trip.sec", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 32679571, "seconds": 1.242}, + "cold": {"reads": 1, "bytes": 331590, "seconds": 17.69}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32.parquet", + "column": "trip.sec", + "file_bytes": 686857703, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 33057447, "seconds": 1.087}, + "cold": {"reads": 1, "bytes": 352939, "seconds": 17.802}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.066}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f64.parquet", + "column": "trip.sec", + "file_bytes": 687450036, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 33068884, "seconds": 1.099}, + "cold": {"reads": 1, "bytes": 352939, "seconds": 17.988}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.067}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "company", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 32679571, "seconds": 1.339}, + "cold": {"reads": 1, "bytes": 1831, "seconds": 0.862}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi.parquet", + "column": "trip.sec", + "file_bytes": 640173859, + "rows": 24314322, + "groups": 25, + "open": {"reads": 27, "bytes": 671548047, "seconds": 3.932}, + "cold": {"reads": 1, "bytes": 6660483, "seconds": 1.077}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.067}, + "arrow_equal": true + } + ] +} diff --git a/bench/remote_parquet_files.py b/bench/remote_parquet_files.py new file mode 100644 index 000000000..d99f3dced --- /dev/null +++ b/bench/remote_parquet_files.py @@ -0,0 +1,194 @@ +"""Measure lazy reads of local Parquet files directly or through localhost HTTP.""" + +import argparse +import http.server +import json +import math +import threading +import time +from contextlib import contextmanager, nullcontext +from email.utils import formatdate +from pathlib import Path + +import pyarrow.compute as pc +import pyarrow.parquet as pq + +import blosc2 + + +@contextmanager +def local_http_server(path): + counts = [0, 0, 0] # GET requests, GET body bytes, HEAD requests. + lock = threading.Lock() + size = path.stat().st_size + + class Ranged(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def headers_for_file(self, length): + self.send_header("Content-Length", str(length)) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) + + def do_HEAD(self): + with lock: + counts[2] += 1 + self.send_response(200) + self.headers_for_file(size) + self.end_headers() + + def do_GET(self): + span = self.headers.get("Range") + first, last = 0, size - 1 + if span: + start, _, end = span.removeprefix("bytes=").partition("-") + if start: + first = int(start) + last = min(int(end), size - 1) if end else size - 1 + else: + first = max(0, size - int(end)) + if first > last: + self.send_response(416) + self.send_header("Content-Range", f"bytes */{size}") + self.end_headers() + return + self.send_response(206 if span else 200) + self.headers_for_file(last - first + 1) + if span: + self.send_header("Content-Range", f"bytes {first}-{last}/{size}") + self.end_headers() + sent = 0 + with path.open("rb") as source: + source.seek(first) + remaining = last - first + 1 + while remaining: + data = source.read(min(1 << 20, remaining)) + if not data: + break + self.wfile.write(data) + sent += len(data) + remaining -= len(data) + with lock: + counts[0] += 1 + counts[1] += sent + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Ranged) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + + def snapshot(): + with lock: + return tuple(counts) + + yield f"http://127.0.0.1:{server.server_port}/{path.name}", snapshot + finally: + server.shutdown() + server.server_close() + thread.join() + + +def measure(path, column=None, *, open_only=False, http=False): + path = Path(path) + context = local_http_server(path) if http else nullcontext((path, None)) + with context as (source, server_snapshot): + started = time.perf_counter() + options = {"block_size": 4096, "cache_type": "none"} if http else None + with blosc2.open(source, storage_options=options) as remote: + snapshot = server_snapshot or (lambda: (remote.traffic.requests, remote.traffic.nbytes, 0)) + opened = snapshot() + open_seconds = time.perf_counter() - started + column = column or remote.col_names[0] + if open_only: + return { + "file": path.name, + "file_bytes": path.stat().st_size, + "rows": len(remote), + "column": column, + "transport": "http" if http else "file", + "open_reads": opened[0], + "open_bytes": opened[1], + "open_heads": opened[2], + "open_seconds": round(open_seconds, 3), + } + started = time.perf_counter() + cold_value = remote[column][-1] + cold_seconds = time.perf_counter() - started + after_cold = snapshot() + cold = tuple(a - b for a, b in zip(after_cold, opened, strict=True)) + started = time.perf_counter() + warm_value = remote[column][-1] + warm_seconds = time.perf_counter() - started + warm = tuple(a - b for a, b in zip(snapshot(), after_cold, strict=True)) + result = { + "file": path.name, + "file_bytes": path.stat().st_size, + "rows": len(remote), + "column": column, + "transport": "http" if http else "file", + "open_reads": opened[0], + "open_bytes": opened[1], + "open_heads": opened[2], + "open_seconds": round(open_seconds, 3), + "cold_reads": cold[0], + "cold_bytes": cold[1], + "cold_heads": cold[2], + "cold_seconds": round(cold_seconds, 3), + "warm_reads": warm[0], + "warm_bytes": warm[1], + "warm_heads": warm[2], + "warm_seconds": round(warm_seconds, 3), + "cache_bytes": remote.cache_bytes, + "warm_equal": bool( + cold_value == warm_value + or (isinstance(cold_value, float) and math.isnan(cold_value) and math.isnan(warm_value)) + ), + } + parquet = pq.ParquetFile(path) + result["groups"] = parquet.num_row_groups + if column in parquet.schema_arrow.names: + last = parquet.read_row_group(parquet.num_row_groups - 1, columns=[column]) + expected = last.column(0)[-1].as_py() + elif parquet.schema_arrow.names == [""]: + root = parquet.read_row_group(parquet.num_row_groups - 1, columns=[""]).column(0).combine_chunks() + leaf = pc.list_flatten(root) + for part in column.split("."): + leaf = leaf.field(part) + expected = leaf[-1].as_py() + else: + expected = None + if expected is not None: + result["arrow_equal"] = bool( + cold_value == expected + or (isinstance(cold_value, float) and math.isnan(cold_value) and math.isnan(expected)) + ) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="+", type=Path) + parser.add_argument("--column") + parser.add_argument("--open-only", action="store_true") + parser.add_argument( + "--http", action="store_true", help="serve each file over localhost HTTP with byte ranges" + ) + args = parser.parse_args() + failed = False + for parquet_path in args.paths: + try: + print( + json.dumps( + measure(parquet_path, args.column, open_only=args.open_only, http=args.http), default=str + ), + flush=True, + ) + except Exception as error: + failed = True + print( + json.dumps({"file": parquet_path.name, "error": f"{type(error).__name__}: {error}"}), + flush=True, + ) + if failed: + raise SystemExit(1) diff --git a/bench/remote_parquet_synthetic_optimized.txt b/bench/remote_parquet_synthetic_optimized.txt new file mode 100644 index 000000000..70dd378b4 --- /dev/null +++ b/bench/remote_parquet_synthetic_optimized.txt @@ -0,0 +1,5 @@ +scalar: file=109920 B open=1 reads/65536 B/0.004 s cold=1 reads/5364 B/0.108 s warm=0 reads/0 B/0.001 s cache=138536 B python_peak=386880 B +wide: file=1096185 B open=1 reads/65536 B/0.026 s cold=1 reads/5364 B/0.118 s warm=0 reads/0 B/0.002 s cache=138536 B python_peak=275013 B +strings: file=64961 B open=1 reads/64961 B/0.001 s cold=1 reads/6352 B/0.007 s warm=0 reads/0 B/0.002 s cache=67367 B python_peak=132149 B +dictionary: file=8231 B open=1 reads/8231 B/0.001 s cold=1 reads/701 B/0.006 s warm=0 reads/0 B/0.001 s cache=134786 B python_peak=304604 B +unnamed-root: file=55530 B open=11 reads/109198 B/0.003 s cold=1 reads/5368 B/0.003 s warm=0 reads/0 B/0.001 s cache=3283 B python_peak=8441705 B diff --git a/bench/remote_parquet_traffic.py b/bench/remote_parquet_traffic.py new file mode 100644 index 000000000..88d346323 --- /dev/null +++ b/bench/remote_parquet_traffic.py @@ -0,0 +1,62 @@ +"""Count in-memory Parquet read operations and bytes for representative tables.""" + +import io +import time +import tracemalloc + +import fsspec +import pyarrow as pa +import pyarrow.parquet as pq + +import blosc2 + + +def measure(name, table, row_group_size, column): + output = io.BytesIO() + pq.write_table(table, output, row_group_size=row_group_size) + blob = output.getvalue() + url = f"memory:///remote-parquet-{name}.parquet" + fsspec.filesystem("memory").pipe(f"/remote-parquet-{name}.parquet", blob) + tracemalloc.start() + started = time.perf_counter() + with blosc2.open(url) as remote: + open_time = time.perf_counter() - started + _, open_peak = tracemalloc.get_traced_memory() + tracemalloc.reset_peak() + opened = (remote.traffic.requests, remote.traffic.nbytes) + started = time.perf_counter() + remote[column][-1] + cold_time = time.perf_counter() - started + _, cold_peak = tracemalloc.get_traced_memory() + tracemalloc.reset_peak() + cold = (remote.traffic.requests - opened[0], remote.traffic.nbytes - opened[1]) + cache_bytes = remote.cache_bytes + started = time.perf_counter() + remote[column][-1] + warm_time = time.perf_counter() - started + warm = (remote.traffic.requests - opened[0] - cold[0], remote.traffic.nbytes - opened[1] - cold[1]) + tracemalloc.stop() + print( + f"{name}: file={len(blob)} B open={opened[0]} reads/{opened[1]} B/{open_time:.3f} s " + f"cold={cold[0]} reads/{cold[1]} B/{cold_time:.3f} s " + f"warm={warm[0]} reads/{warm[1]} B/{warm_time:.3f} s " + f"cache={cache_bytes} B python_peak={max(open_peak, cold_peak)} B" + ) + + +if __name__ == "__main__": + n = 10_000 + measure("scalar", pa.table({"x": list(range(n)), "y": list(range(n))}), 1_000, "x") + measure("wide", pa.table({f"c{i}": list(range(n)) for i in range(20)}), 1_000, "c0") + measure("strings", pa.table({"s": [f"value-{i:05d}" for i in range(n)]}), 1_000, "s") + measure( + "dictionary", + pa.table({"d": pa.array([f"v{i % 100}" for i in range(n)]).dictionary_encode()}), + 1_000, + "d", + ) + root = pa.Table.from_arrays( + [pa.array([[{"x": i}] for i in range(n)], type=pa.list_(pa.struct([("x", pa.int64())])))], + names=[""], + ) + measure("unnamed-root", root, 1_000, "x") diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index 31742d083..9489fe1c7 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -5,6 +5,58 @@ Fixed-width, `blosc2.utf8()`, batch-backed variable-length, list, struct/object, and dictionary columns are fetched on demand, including their null masks. A table inside a hierarchy can also be opened through `RemoteStore`. +## Single-file Parquet (experimental) + +`blosc2.open()` recognizes `.parquet` paths, including fsspec URLs, and returns +a read-only `RemoteCTable`. Use `source_format="parquet"` for an extensionless +URL. `storage_options` go to fsspec; `parquet_options` go to PyArrow's +`ParquetFile`. `columns`, `max_rows`, null policy, string width, and list +conversion options follow `CTable.from_parquet()`. + +```python +with blosc2.open( + "s3://bucket/readings.parquet", + storage_options={"anon": True}, + columns=["station", "temperature"], + parquet_options={"read_dictionary": ["station"]}, + cache_dir="parquet-cache", +) as table: + last = table["temperature"][-1] + local = table.copy(urlpath="readings.b2z") +``` + +Reads convert the requested physical field and row group using the Arrow +importer. A narrow scalar read can avoid unrelated fields and groups; a group +with an indivisible large value can still allocate much more than the cache +budget. With mask-backed nulls, schema inference reads only the footer; in-band +null policies may need a first-batch sample. Flattening a single unnamed +`list` root reads the smallest leaf from each group to establish the +logical row count. +`max_rows` stops that preparation at the requested prefix. `cache_dir` retains +converted groups and the row map for later opens; without it, an in-memory LRU +is used. `refresh()` reopens the source and invalidates old views. `lazy=False` +calls the eager `CTable.from_parquet()` importer. +Reads against the shared seekable Parquet handle are serialized; increasing +`max_concurrency` does not make one file's row-group reads parallel. +HTTP servers must honor byte-range requests; fsspec raises a range-request error +for servers that only return complete files. Download the file and import it +locally when range access is unavailable. + +Parquet disk caches require source size and an ETag or modification time. +`table.save("reference.b2nd")` includes already converted groups and the row +map; use `include_cache=False` for a small cold reference. `blosc2.open()` +reopens a reference when the source marker still matches. Runtime credentials +and reader objects are omitted. Reopen a protected source with +`blosc2.RemoteCTable.open_reference("reference.b2nd", storage_options={...})`. +`shared_cache=True` permits separate processes to reuse a `cache_dir`; it +requires a disk cache and serializes cache lookup and publication with a file +lock. Every process sharing that directory should use the same source and +conversion options. `RemoteCTable.with_sparse_cache(url, cache_dir)` uses the +same Parquet cache for a single `.parquet` source. +The `traffic` counter reports file-handle reads and bytes returned; +for an HTTP or object-store driver, its own buffering can change actual wire +traffic. + `blosc2.open()` dispatches uncached local B2Z table archives to `CTable`, and remote B2Z archives and selected local or remote PyTables tables to `RemoteCTable`. Supplying `cache_dir=` also selects `RemoteCTable` for a local diff --git a/plans/remote-parquet.md b/plans/remote-parquet.md new file mode 100644 index 000000000..dca684e80 --- /dev/null +++ b/plans/remote-parquet.md @@ -0,0 +1,541 @@ +# Remote Parquet tables: implementation plan + +Status: implementation in progress; final parity gate remains open. +Baseline: Python-Blosc2 working tree inspected on 2026-09-25. + +## Goal and completion contract + +Support read-only remote Parquet tables through the existing fsspec/remote-table +API, with at least the data types, conversions, selection options, and logical +results currently supported by `CTable.from_parquet()`. Implement this in small, +independently tested iterations. A numeric-only prototype is an intermediate +milestone, not completion of this goal. + +For a supported single Parquet file and equivalent conversion options, compare +the remote table with `CTable.from_parquet()` on the same file. Require matching +logical schema, column order, row count, values, nulls, nested row reconstruction, +query results, and Arrow export semantics. Match existing documented conversion +and rejection behavior; this does not promise lossless handling of types or +metadata that the current importer already transforms or rejects. + +Ordinary opening should read metadata alone for mask-backed nulls; in-band null +policies may still require an inference sample. Row reads should fetch the necessary physical column chunks +and row groups, without importing the entire table. Some existing conversions +need more preparation, particularly unnamed-root flattening and dictionary +normalization. Account for and document those reads rather than claiming all +files can open from the footer alone. + +Preserve `CTable.from_parquet()` as the eager, batched import API. Destination +creation options apply when materializing a remote table; a remote source remains +read-only. No silent eager-import fallback for unsupported lazy cases. + +## Existing implementation to reuse + +- `src/blosc2/ctable.py`: `from_parquet()`, `from_arrow()`, Arrow schema compilation, + null policy, nested flattening, and batch conversion. The importer passes its + input directly to `pyarrow.parquet.ParquetFile` and forwards reader kwargs. +- `src/blosc2/remote_ctable.py`: read-only table API, lifetime management, tuning, + persistent carriers, and materialization behavior. +- `src/blosc2/remote_store.py`: format discovery, shared source ownership, + generation checks, filesystem access, and table discovery. +- `src/blosc2/ctable_storage.py`: `RemoteTableStorage`, synthetic all-valid rows, + fixed and variable column access, and the existing HDF5-specific integration. +- `src/blosc2/ctable_remote_read.py`: projected/gathered table reads and bounded + parallel work. Audit its assumptions about Blosc2 arrays and batch backends. +- `src/blosc2/remote_batch.py`, `remote_store_cache.py`, and + `remote_source_cache.py`: batch access, cache ownership/accounting, and source + metadata reuse. Reuse compatible pieces; a Parquet column chunk is not a B2Z + member and cannot simply inherit its addressing rules. +- `src/blosc2/schunk.py`, `core.py`, and `ref.py`: opening, URL parsing, source + descriptors, and reopening. Trace all format allowlists and serialized-source + validators before adding the new format. +- `pyproject.toml`: `blosc2[parquet]` already supplies PyArrow. fsspec and protocol + drivers remain the transport dependencies. + +A prior read-only probe verified `CTable.from_parquet(file_handle)` and +`CTable.from_parquet(path, filesystem=fs)` with fsspec's memory filesystem. This +establishes transport compatibility for eager import, not lazy access or HTTP/S3 +performance. The CLI additionally assumes local `Path` inputs; changing it is +not required for the lazy backend. + +## Compatibility inventory + +Turn this inventory into executable cases in iteration 1. Use the actual eager +result as the reference, including edge cases where physical Arrow types change +after a Parquet round trip. + +| Capability | Required outcome | Main implementation concern | +| --- | --- | --- | +| Signed/unsigned integers, float32/64, Boolean | Same values, dtypes, operations | Row-group to logical-row mapping; boundaries and NaNs | +| Timestamps | Same units, timezone metadata, nulls, comparisons | Share importer conversion rules | +| UTF-8 and binary, including supported large/fixed variants | Same logical strings/bytes and empty values | Variable payload sizes and offset addressing | +| `string_max_length` scalar/mapping | Same fixed-width selection and overflow errors | Freeze inferred schema before out-of-order reads | +| Nullable scalars | Same mask/sentinel behavior and null-policy precedence | Keep valid sentinel-like values distinct in mask mode | +| Dictionary strings | Same supported categorical semantics | Dictionaries/codes can differ between row groups | +| Fixed-size lists / shaped cells | Same mapped CTable shape and values | Field metadata, row-level and child null semantics | +| Lists and large lists | Same list values and supported children | Variable element counts; null versus empty list | +| Nested lists and list-of-struct | Same nested values | Both `msgpack` and `arrow` list serializers | +| Top-level structs | Same flattened field paths and reconstructed rows | Parent validity, escaped names, leaf projection | +| Unnamed fields | Same collision-safe root renaming | Store reversible physical/logical name mapping | +| Single unnamed list-of-struct root | Same flattened element rows by default | Footer row counts describe outer rows, not output rows | +| `separate_nested_cols=False` | Same non-separated representation | Keep physical outer-row semantics | +| `columns` | Same accepted names, order, and errors | Import-time selection versus later leaf projection | +| `max_rows` | Same prefix, including zero | Applies after unnamed-root flattening | +| Empty data | Same schema and empty results | No first batch available for inference | +| Unsupported types | Same deliberate rejection | No new implicit object fallback | +| Reader kwargs | Equivalent PyArrow decoding behavior | Separate transport, conversion, and reader options | +| Batch/compression/validation options | Same logical conversion behavior | Apply storage tuning to cache/materialization appropriately | +| Schema and nested metadata | Same metadata semantics as eager import | Do not accidentally promise extra CLI-only metadata handling | + +The authoritative starting tests are `tests/ctable/test_parquet_interop.py`, +`test_arrow_interop.py`, and `test_null_mask_arrow.py`. Arrow-only cases become +requirements only where the same case is accepted by `from_parquet()` after +writing/reading Parquet. In particular, `from_arrow(object_fallback=True)` is not +a capability of `from_parquet()`. + +## Proposed public API + +Use `blosc2.open(url, source_format="parquet", ...)` for a single table at the +file root, returning `RemoteCTable`. Recognize `.parquet` automatically, including +URLs with query parameters; explicit format handles extensionless URLs. Preserve +the existing `lazy` convention, and document both lazy and explicit eager paths. +Allow direct `RemoteCTable` construction using the same source options. + +Illustrative target API, subject to the iteration-1 keyword audit: + +```python +table = blosc2.open( + "s3://bucket/data.parquet", + source_format="parquet", + lazy=True, + storage_options={"anon": True}, + columns=["temperature", "station"], + cache_dir="cache", + parquet_options={"read_dictionary": ["station"]}, +) +``` + +Keep `storage_options` for fsspec credentials/configuration. Put PyArrow reader +kwargs in a dedicated `parquet_options` mapping to avoid collisions with +`blosc2.open()` and conversion parameters. Expose conversion parameters matching +the importer where relevant: `columns`, `max_rows`, `string_max_length`, +`null_storage`, `auto_null_sentinels`, `separate_nested_cols`, `list_serializer`, +and batch settings. Resolve the exact naming once, in iteration 1. + +`urlpath`/`mode` used as importer destination options belong to materialization, +not source mutation. Keep compression/decompression and validation settings +usable for converted caches and materialized output. Do not silently ignore +options that lack a remote meaning. Reader settings requiring local files, such +as memory mapping, need explicit documented behavior on remote handles. + +Credentials, live filesystems, decryption objects, and other runtime-only reader +objects must not be serialized into portable carriers. Direct reads may use +them; reopening must accept replacement runtime options or fail clearly. Resolve +ambient null policy when opening and record the effective conversion policy so +later reads cannot change with a different context. + +## Internal approach + +Add a focused Parquet source module rather than a new general storage framework. +One owner holds the filesystem handle, `ParquetFile`, footer/schema, row-group +boundaries, effective conversion options, and shared cache state. Expose the file +as a table node through the existing remote ownership model. + +Map requested logical rows to row groups using cumulative counts. Group gathers +by physical group, read the union of requested physical fields once, convert +through shared Arrow/CTable rules, and restore caller order and duplicates. +Separate logical leaf names from Parquet physical paths. Projected nested fields +may need parent validity or other structural data; request that explicitly. + +Keep Parquet knowledge at this source/storage boundary. Existing CTable queries +and exporters should consume ordinary column semantics. Use the existing column +and batch interfaces where they fit, with the smallest necessary adapter where +they do not. Prototype both scalar and variable-length access before fixing the +internal interface: global UTF-8 offsets and dictionary vocabularies are not +available by pretending each Parquet group is an independent whole table. + +Prefer converted Blosc2 cache units keyed by physical row group and logical +column/conversion identity. Keep related validity and dictionary dependencies +consistent. Transient Arrow buffers must be bounded and separately accounted +for. Avoid repeatedly decoding one large group for adjacent small Blosc2 chunks. +Do not build a second persistent raw-byte cache unless measurement shows a need. + +## Iterations and acceptance gates + +Each iteration leaves a usable, tested increment and updates this document with +completed scope, remaining restrictions, and measured behavior. Complete gates in +order; failures in schema or row semantics block performance work. + +### 1. Freeze the compatibility contract and prove the storage seam + +- Inventory the importer tests and produce a compact parameterized parity matrix. + Include schema, null policy, metadata, reader kwargs, and invalid inputs. +- Trace source opening, remote storage, column reads, query evaluation, exports, + serialization, and close/refresh paths. Record format-specific assumptions. +- Prototype one numeric column and one UTF-8/list column through the proposed + storage seam. Determine whether to extend `RemoteTableStorage` directly or add + a small Parquet-specific storage implementation sharing its lifecycle. +- Extract only shared importer normalization/conversion code that the prototype + actually needs. Preserve eager behavior and avoid duplicating type mappings. +- Resolve API keyword routing, capability errors during intermediate stages, and + metadata-versus-sample requirements. Keep experimental APIs private until stable. + +Gate: representative remote reads match eager import, existing import tests pass, +and the chosen storage seam supports both fixed and variable payloads without +whole-file import. Record any data-dependent schema inference discovered. + +### 2. Single-file lazy access for scalar columns + +- Open local and fsspec sources, obtain footer/schema, and expose a table node. +- Implement numeric, Boolean, timestamp, and nullable scalar columns, synthetic + all-valid rows, projection, ordinary `max_rows`, and empty tables. +- Implement row indexing/slicing/gathers to the extent supported by the existing + CTable API, including negative indices, reordered/duplicate positions, empty + selections, bounds errors, and reads spanning unequal row groups. +- Integrate normal CTable scan queries and basic Arrow/materialized output. +- Own/close file handles correctly; reject writes and unsupported types clearly. + Add initial shared in-memory cache and source generation checks. + +Gate: scalar parity and scan results pass. Instrumented reads prove a narrow +selection does not scan unrelated row groups or columns, allowing documented +footer/prefetch overhead. Close and failure paths leak no owned handles. + +### 3. Strings, binary, and dictionary columns + +- Implement UTF-8, supported binary variants, fixed-width overrides, and nulls. + Preserve empty string/bytes separately from missing values. +- Reuse width inference exactly where the importer relies on it; inference must + be deterministic, independent of which row the user accesses first. +- Supply lazy variable-length column access without building global payload + offsets by scanning all strings at open. +- Handle dictionary evolution and dictionary order differences across groups. + Reuse the importer's dictionary semantics. If the CTable representation needs + a global vocabulary, perform an explicit bounded-memory preparation pass over + the affected column, cache its result, and account for its cost. Never expose + group-local codes as if they were global or mutate published code meanings. + +Gate: round trips, filtering, null masks, multibyte strings, binary zero bytes, +overflow errors, dictionaries across multiple groups, and out-of-order reads +match eager import. Warm reads reuse converted data within the cache budget. + +### 4. Structs, shaped cells, and ordinary nested lists + +- Implement top-level struct flattening, field-name escaping, reconstruction, + and logical-leaf to physical-field projection using shared importer helpers. +- Implement fixed-size lists/shaped cells and all list/nested combinations + accepted by the importer, with both serializers and batch-setting variants. +- Preserve parent and child validity as the importer does. Cover null structs, + lists containing nulls, empty lists, null lists, and nested timestamps. +- Support unnamed-field renaming and `separate_nested_cols=False` behavior. +- Verify batch boundaries independent of row-group boundaries and large cells + exceeding nominal buffer targets. Document indivisible allocation limits. + +Gate: the ordinary-row portion of the complete parity matrix passes, including +nested queries, gathered reads, Arrow export, and materialization/reopen. + +### 5. Flattened unnamed-root row semantics + +This is required for the goal, not deferred nested-data polish. + +- Reuse detection and flattening for a single unnamed list/large-list of struct. + Distinguish physical outer rows from logical element rows everywhere. +- Build cumulative logical element counts per physical row group. PyArrow may + have to decode nested data to recover the required lengths; do not assume + footer counts or a cheap offsets-only API exist. +- Start with a bounded-memory preparation scan of the required nested data when + exact `len()` and random indexing need a complete map. Persist this map by + source generation and conversion policy. Reuse decoded groups when practical. +- If `max_rows` bounds the logical prefix, stop preparation once that prefix is + known, or at EOF when the source contains fewer elements. Zero must yield the + correct empty schema without a payload scan unless inference truly requires it. +- Match null/empty outer-list handling, field metadata, reconstructed rows, and + the different semantics when separation is disabled. + +Gate: logical row counts and values match eager import across group boundaries, +null/empty outer rows, reordered reads, and limits. Reopening a valid cached map +avoids repeating preparation. Preparation traffic is reported separately. + +### 6. Persistent cache, source identity, and portable reopening + +- Extend existing cache/carrier descriptors and format validators for Parquet. + Include format version, normalized source identity/generation, column mapping, + selection, effective null/conversion policy, reader semantics, and row-map + identity. Include storage representation settings where they affect cache data. +- Follow the existing immutable-source/explicit-refresh contract. Refresh + invalidates dependent schemas, row maps, dictionaries, and payloads together; + old views/columns must fail as stale rather than mix generations. +- Support the existing applicable cache policies, aggregate byte budget, eviction, + disk reuse, and shared sparse-cache entry points. Eviction must not leave data + without its required validity/dictionary dependencies. +- Publish converted units and preparation metadata atomically, using existing + locks/publication mechanisms. Interrupted or corrupt caches rebuild safely. +- Implement carrier save/reopen, materialization to ordinary writable CTable, + runtime filesystem/credential reinjection, and clear failures for nonportable + reader options. Keep source credentials out of serialized metadata and logs. +- Exercise concurrent reads and shared ownership; serialize access to a shared + seekable Python handle unless safe concurrent access is established. + +Gate: cold/warm/reopened results agree, retained-cache reads avoid extra payload +fetches, limits/eviction work, source refresh cannot mix generations, and failed +publication cannot expose partial tables. Test memory and disk policies as well +as shared-cache behavior supported by the final API. + +### 7. Complete the parity gate and public integration + +- Finish `blosc2.open`, direct constructor, extensionless URL, format detection, + source descriptor, and materialization option routing. Ensure `.parquet` + detection does not disturb Zarr/HDF5/B2Z opening or signed URL handling. +- Run the full compatibility matrix against eager imports, with each current + capability assigned a passing case. Cover reader kwargs such as dictionary + decoding and timestamp coercion; validate transport-incompatible combinations. +- Exercise normal column access, projections, views, filters/reductions where + supported, Arrow streaming, Parquet export, and local persistence/reopen. + Queries may scan; predicate pushdown is not needed for semantic parity. +- Add a counted HTTP range-server test to validate real seek/range behavior and + a small optional S3-compatible network smoke test. Document servers without + range support: either require explicit download/import or fail clearly. +- Measure open/preparation time, requests/bytes, peak temporary memory, cold and + warm reads, and conversion/cache cost for representative scalar, wide, + variable-length, dictionary, and unnamed-root files. +- Add user documentation and runnable examples explaining lazy reads, projected + columns, preparation costs, cache behavior, materialization, and reader options. + +Gate: all baseline capabilities are supported with documented remote costs; +existing import and remote-table regression tests pass. Report any newly found +baseline bugs separately and agree their resolution before calling parity done. + +## Validation strategy + +Use the required `blosc2` conda environment. Keep new coverage in a focused +`tests/ctable/test_remote_parquet.py`, reusing existing fixtures/helpers where +practical. Parameterize meaningful compatibility cases rather than copying the +entire importer suite. Add format-routing/cache tests to their existing modules +only where those shared contracts change. + +For each fixture, write Parquet locally, import it eagerly, expose the same bytes +through fsspec, and compare remote behavior. Compare null-aware values and logical +schema explicitly; plain equality is insufficient for NaN, nested nulls, and +dictionary encoding. Test nonuniform row groups and mismatched Arrow/Blosc2 batch +sizes to expose accidental alignment assumptions. + +Instrument both transport reads and decoder calls: byte caching can hide repeated +decoding, and decoder counters can hide excessive network prefetch. Use structural +assertions about touched groups/fields and bounded buffers, not brittle exact +request counts tied to one PyArrow release. Large indivisible values and decoded +Arrow allocation overhead must be reported rather than hidden behind a compressed +cache budget. + +Run focused tests after each iteration. At final integration, run the relevant +Parquet/Arrow/null-mask suites plus remote CTable/store/array regressions and the +repository's required checks. Network tests should remain explicitly marked and +must not be necessary for deterministic local correctness coverage. + +## Scope after the goal + +The following can be separate iterations after single-file importer parity: + +- Partitioned/multi-file datasets, directory discovery, and schema unification. +- Parquet statistics/page-index predicate pushdown and native index generation. +- Remote Parquet writes, updates, or transactional mutation. +- Caterva2 server/UI discovery and serving. This needs a separate cross-repository + pass, including format allowlists and source validation; Python-Blosc2 support + alone does not establish Caterva2 support. +- CLI URL convenience, specialized prefetch, and page-level decoding optimization. + +These exclusions do not remove any single-file data capability currently accepted +by `CTable.from_parquet()`. Additional unsupported types or implicit object +fallbacks are outside the compatibility target unless requested separately. + +## Progress ledger + +Prototype status (2026-09-25): `src/blosc2/remote_parquet.py` provides a read-only +row-group adapter for local and fsspec single files. `blosc2.open()` and direct +`RemoteCTable` construction recognize Parquet, with explicit eager opening via +`lazy=False`. It uses `CTable.from_arrow()` for schema compilation and for +on-demand conversion of physical fields, so the tested scalar, UTF-8, list, +struct, fixed-size-list, dictionary, and flattened unnamed-root results match +`CTable.from_parquet()`. Explicit refresh invalidates old views. `cache_dir` +retains converted row groups and flattened-root row maps across opens, with a +byte budget and corrupt-entry rebuilding. File locking permits concurrent +processes to reuse a disk cache. CFrame-backed references can retain converted +groups and logical row maps, reopen through `blosc2.open()`, and accept +replacement runtime options through `RemoteCTable.open_reference()`. A local HTTP range-server test confirms +that a narrow cold read transfers less than the file and a warm read transfers +zero bytes. After the request optimizations, the focused Arrow/Parquet, +remote-table, and open regressions pass (458 cases). The full default suite +passed (10,633 passed, 36 skipped) before these optimizations. This is an +intermediate increment. + +Current limits: in-band null policies may sample the first row group; flattened +unnamed roots scan one leaf per group for exact logical counts. Converted +groups are retained in an in-memory LRU or source-identified disk files. +Credential replacement is covered with a runtime fsspec option, but an actual +authenticated service is not yet tested. More importer edge cases, reader-option +routing, memory measurements, and optional S3 smoke coverage remain. +The first attempt to add nullable fixed-size-list parity exposed a baseline +PyArrow `iter_batches()` error when reading that Parquet file: "Expected all lists +to be of size=2 but index 2 had size=0." The eager importer fails before remote +comparison; the valid fixed-size-list variant passes. Nullable fixed-size lists +remain outside remote parity while the eager importer cannot read them. +These missing pieces keep the final gate open. + +The in-memory benchmark `bench/remote_parquet_traffic.py` counts file-handle +read calls and bytes returned, which approximate requests/transfer for this +transport but do not model HTTP or S3 buffering. With 10,000 rows in ten row +groups, the pre-optimization baseline measured: + +| Fixture | Parquet bytes | Open reads / bytes | Cold final-row reads / bytes | Warm reads / bytes | +| --- | ---: | ---: | ---: | ---: | +| 2-column scalar | 109,920 | 1 / 65,536 | 1 / 5,364 | 0 / 0 | +| 20-column wide | 1,096,185 | 1 / 65,536 | 1 / 5,364 | 0 / 0 | +| UTF-8 strings | 64,961 | 1 / 64,961 | 1 / 6,352 | 0 / 0 | +| Dictionary strings | 8,231 | 2 / 15,241 | 1 / 701 | 0 / 0 | +| Flattened unnamed root | 55,530 | 12 / 162,866 | 1 / 5,368 | 0 / 0 | + +Open bytes may exceed file size because the in-memory handle returns overlapping +footer and sampled-group reads. Scalar and UTF-8 opens use footer-only schema +compilation. Dictionary opening samples one row group. The unnamed-root open +includes its logical-row preparation scan. + +The reproducible script now also times each stage and reports converted-cache +bytes and `tracemalloc`'s peak Python allocation. On this host, traced peaks +were 494,333 B (scalar), 1,371,630 B (wide), 197,234 B (strings), 314,839 B +(dictionary), and 155,616 B (unnamed root). These exclude Arrow's native buffers +and are not a bound on process RSS or indivisible row-group allocation. + +The counted localhost HTTP range server, serving a 119,757-byte two-column +file with ten row groups, returned 1 request / 65,536 bytes on open, +1 request / 5,364 bytes for a cold `x[9500]`, and 0 / 0 for the warm repeat +with `storage_options={"block_size": 4096, "cache_type": "none"}`. These are +server-side GET response totals, unlike the in-memory file-handle figures above; +metadata HEAD requests are separate. A second localhost test verifies that a +server ignoring ranges fails with fsspec's range-request error. An optional +`network`-marked S3 smoke test uses `BLOSC2_REMOTE_PARQUET_S3_URL`. + +### Chicago taxi files (local range-read baseline) + +`bench/remote_parquet_files.py` measures the same cold and warm access stages +against existing files without copying them. It compares the last-row value +with PyArrow's last row group. These are **local file-handle read calls and bytes +returned**, not HTTP or S3 wire requests. Times include row-group conversion. +The exact measurements are saved in `bench/remote_parquet_chicago_results.json`. +The measurements below use the default in-memory cache and `trip.sec` unless +another column is named. + +| File / column | File bytes | Logical rows | Open calls / bytes / s | Cold calls / bytes / s | Warm calls / bytes / s | Arrow value | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 581,387,271 | 24,314,322 | 2 / 32,679,571 / 1.242 | 1 / 331,590 / 17.690 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 686,857,703 | 24,314,322 | 2 / 33,057,447 / 1.087 | 1 / 352,939 / 17.802 | 0 / 0 / 0.066 | match | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 687,450,036 | 24,314,322 | 2 / 33,068,884 / 1.099 | 1 / 352,939 / 17.988 | 0 / 0 / 0.067 | match | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 581,387,271 | 24,314,322 | 2 / 32,679,571 / 1.339 | 1 / 1,831 / 0.862 | 0 / 0 / 0.069 | match | +| `chicago-taxi.parquet` / `trip.sec` | 640,173,859 | 24,314,322 | 27 / 671,548,047 / 3.932 | 1 / 6,660,483 / 1.077 | 0 / 0 / 0.067 | match | + +The original `chicago-taxi.parquet` is a single unnamed nested list-of-struct +field. Its footer has 7,728 outer rows in 25 groups, and the lazy flattened +view opens with 24,314,322 logical rows. Opening reads 27 times and returns +671,548,047 bytes in 3.932 seconds because it scans every group to build the +logical-row map. An initial attempt exposed an expensive schema probe that +converted the entire first group; the probe now uses one outer row. The +synthetic parity suite still passes after this change. Its last `trip.sec` +value matches the nested Arrow leaf in the final physical group. + +### Chicago taxi files over localhost HTTP + +The same five measurements were repeated with +`bench/remote_parquet_files.py --http`. Its localhost server honors byte ranges +and counts completed GET responses and response-body bytes at the server. The +fsspec client uses `block_size=4096` and `cache_type="none"`; converted row groups +remain in the default in-memory cache. The exact results are saved in +`bench/remote_parquet_chicago_http_results.json`. Each open also made two HEAD +requests; cold and warm reads made no HEAD requests. Bytes exclude HTTP headers +and transport overhead. Times include local loopback transfer and conversion. + +| File / column | Open GETs / bytes / s | Cold GETs / bytes / s | Warm GETs / bytes / s | Arrow value | +| --- | ---: | ---: | ---: | --- | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 2 / 32,679,571 / 1.253 | 1 / 331,590 / 17.688 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 2 / 33,057,447 / 1.084 | 1 / 352,939 / 17.885 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 2 / 33,068,884 / 1.122 | 1 / 352,939 / 18.131 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 2 / 32,679,571 / 1.308 | 1 / 1,831 / 0.813 | 0 / 0 / 0.068 | match | +| `chicago-taxi.parquet` / `trip.sec` | 27 / 671,548,047 / 3.909 | 1 / 6,660,483 / 1.051 | 0 / 0 / 0.067 | match | + +The HTTP GET bytes matched the local file-handle bytes for each phase. The +unnamed nested root remains expensive to open because the exact flattened row +count requires reading all groups. For the flat numeric files, the roughly +18-second cold access is dominated by converting a whole row group, despite a +single sub-megabyte GET for `trip.sec`. + +For `chicago-taxi-flat-f32.parquet`, the final group has 259,680 rows and its +`trip.sec` chunk is 352,939 compressed bytes. Reading and decoding that chunk +directly with PyArrow took about 0.001 seconds from the local file. A `cProfile` +trace of the cold remote access showed 259,680 calls to `CTable._write` and +259,712 calls to `NDArray.__setitem__` during `CTable.from_arrow()` conversion. Profiling raised +the total to 32.4 seconds versus 17.9 seconds uninstrumented. The dominant cost +is Python per-row materialization and index bookkeeping, not the HTTP transfer. +For faster random scalar access, conversion needs a vectorized path or a way to +read the selected value without materializing the entire row group. + +### Request and transfer optimization (2026-09-25) + +The earlier tables are the baseline. The updated localhost results are saved in +`bench/remote_parquet_chicago_http_optimized.jsonl`. Default mask-backed schema +inference now uses the Arrow footer schema without reading a data sample, and +ordinary in-memory opens defer the source-identity lookup until a reference is +saved. This changes flat-file opening from two GETs, about 33 MB and two HEADs +to one 65,536-byte GET and one HEAD. Cold and warm row reads are unchanged. + +For an unnamed `list` root, the exact logical row count still requires +examining every row group. The scanner now projects the smallest compressed +Parquet leaf in each group; that leaf carries the outer list's repetition levels. +The Chicago nested file's open went from 27 GETs / 671,548,047 bytes / 3.909 s +to 26 GETs / 333,139 bytes / 0.364 s. A disk-cached row map reduces a reopened +nested file to one footer GET / 65,536 bytes. The HTTP source marker now accepts +fsspec's `Last-Modified` key, allowing this cache to work with HTTP servers that +provide that header. + +| File / column | Open GETs / bytes / s | Cold GETs / bytes / s | Warm GETs / bytes / s | Arrow value | +| --- | ---: | ---: | ---: | --- | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 1 / 65,536 / 0.188 | 1 / 331,590 / 17.571 | 0 / 0 / 0.068 | match | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 1 / 65,536 / 0.022 | 1 / 352,939 / 17.638 | 0 / 0 / 0.068 | match | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 1 / 65,536 / 0.016 | 1 / 352,939 / 17.862 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 1 / 65,536 / 0.189 | 1 / 1,831 / 0.819 | 0 / 0 / 0.068 | match | +| `chicago-taxi.parquet` / `trip.sec` | 26 / 333,139 / 0.364 | 1 / 6,660,483 / 1.083 | 0 / 0 / 0.068 | match | + +The [fsspec caching strategies](https://filesystem-spec.readthedocs.io/en/latest/features.html) +and [h5py chunk/page buffering](https://docs.h5py.org/en/latest/high/file.html) +motivated a read-ahead check. On the old full-column nested scan, 16, 64 and +128 MiB fsspec read-ahead blocks used 25, 9 and 6 GETs respectively versus 26 +without buffering, but all transferred about 640 MB and took about 3.8–3.9 s. +The leaf projection removes almost all of that transfer, while large read-ahead +would fetch unrelated fields and groups, so it is left as an explicit fsspec +`storage_options` choice rather than a default. The converted-row-group cache +continues to eliminate requests for repeated values. + +The synthetic benchmark was rerun and saved in +`bench/remote_parquet_synthetic_optimized.txt`. Dictionary opening fell from +2 reads / 15,241 bytes to 1 read / 8,231 bytes. The flattened-root open fell +from 12 reads / 162,866 bytes to 11 reads / 109,198 bytes. Scalar, wide, and +UTF-8 open counts stayed at one read. + +- [ ] 1. Compatibility inventory, API contract, and storage seam +- [ ] 2. Scalar lazy access +- [ ] 3. Strings, binary, and dictionaries +- [ ] 4. Structs, shaped cells, and nested lists +- [ ] 5. Unnamed-root flattening and logical row map +- [ ] 6. Persistent caching and portable lifecycle +- [ ] 7. Complete parity, transport verification, and documentation + +Do not estimate completion from the numeric prototype alone. The largest +uncertainties are schema inference shared with the importer, global dictionary +semantics, variable-length column interfaces, and the preparation required for +flattened root rows. Re-estimate remaining work after iterations 1 and 5. + +## Library references + +- [PyArrow ParquetFile](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html): + metadata, selected-column row-group reads, and batch iteration. +- [Arrow Parquet documentation](https://arrow.apache.org/docs/python/parquet.html): + format layout and filesystem integration. + +Use PyArrow for decoding and fsspec for transport. Verify behavior against the +installed/supported versions during implementation, especially nested projection, +reader options, prefetch, and Python-file concurrency. diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index 1f1292c5d..539ce0307 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -128,6 +128,10 @@ def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carri if carrier is None: raise ValueError("A persisted RemoteArray requires its B2ND carrier") return blosc2.RemoteArray._from_payload(payload, carrier) + if kind == "remote_parquet": + from blosc2.remote_parquet import RemoteParquetCTable + + return RemoteParquetCTable._from_payload(payload) if kind == "lazyexpr": return decode_structured_lazyexpr(payload, carrier_path=carrier_path) if kind == "lazyudf": diff --git a/src/blosc2/ctable_remote_read.py b/src/blosc2/ctable_remote_read.py index 493366436..e38752cf3 100644 --- a/src/blosc2/ctable_remote_read.py +++ b/src/blosc2/ctable_remote_read.py @@ -345,6 +345,28 @@ def column_values(table, names, positions, *, null_masks=None): # noqa: C901 ) storage = table._remote_read_storage() + if storage._owner.format == "parquet": + storage._check_open() + result = {} + for name in names: + result[name] = ( + table._cols[name][positions] + if name in table.col_names + else table._fetch_col_at_positions_uncached(name, positions) + ) + if null_masks is not None and name in table.col_names: + mask = table._null_mask(name) + if mask is not None: + null_masks[name] = ~mask[positions] + elif name in table.col_names: + mask = table._null_mask(name) + if mask is not None: + missing = ~mask[positions] + if missing.any(): + result[name] = list(result[name]) + for index in np.flatnonzero(missing): + result[name][index] = None + return result source_columns = _source_columns(table) with storage._owner.lock: storage._check_open() diff --git a/src/blosc2/remote_ctable.py b/src/blosc2/remote_ctable.py index e2909fdb6..dbddb3ab9 100644 --- a/src/blosc2/remote_ctable.py +++ b/src/blosc2/remote_ctable.py @@ -9,7 +9,7 @@ from __future__ import annotations import operator -import os # noqa: TC003 +import os from blosc2.ctable import CTable from blosc2.ctable_storage import RemoteTableStorage @@ -86,14 +86,87 @@ def __new__( cache_policy=CACHE_POLICY_DEFAULT, max_cache_bytes=CACHE_POLICY_DEFAULT, cache_dir=None, + shared_cache=False, hdf5_index=None, max_concurrency=8, metadata_buffer_bytes=8 << 20, row_buffer_bytes=64 << 20, + source_format=None, + parquet_options=None, + columns=None, + max_rows=None, + string_max_length=None, + null_storage=None, + auto_null_sentinels=True, + separate_nested_cols=True, + list_serializer="msgpack", + blosc2_batch_size=2048, + blosc2_items_per_block=None, + batch_size=2048, + cparams=None, + dparams=None, + validate=False, _filesystem=None, _filesystem_resolver=None, _batch_validator=None, ): + if source_format == "parquet" or ( + isinstance(urlpath, (str, os.PathLike)) + and os.fspath(urlpath).split("?", 1)[0].lower().endswith(".parquet") + ): + if source_format not in (None, "parquet"): + raise ValueError("source_format conflicts with the .parquet suffix") + from blosc2.remote_parquet import RemoteParquetCTable + + return RemoteParquetCTable( + os.fspath(urlpath), + storage_options=storage_options, + parquet_options=parquet_options, + columns=columns, + max_rows=max_rows, + string_max_length=string_max_length, + null_storage=null_storage, + auto_null_sentinels=auto_null_sentinels, + separate_nested_cols=separate_nested_cols, + list_serializer=list_serializer, + blosc2_batch_size=blosc2_batch_size, + blosc2_items_per_block=blosc2_items_per_block, + batch_size=batch_size, + cparams=cparams, + dparams=dparams, + validate=validate, + max_cache_bytes=max_cache_bytes, + cache_policy=cache_policy, + cache_dir=cache_dir, + shared_cache=shared_cache, + max_concurrency=max_concurrency, + metadata_buffer_bytes=metadata_buffer_bytes, + row_buffer_bytes=row_buffer_bytes, + ) + if ( + source_format is not None + or any( + value is not None + for value in ( + parquet_options, + columns, + max_rows, + string_max_length, + null_storage, + blosc2_items_per_block, + cparams, + dparams, + ) + ) + or auto_null_sentinels is not True + or separate_nested_cols is not True + or list_serializer != "msgpack" + or blosc2_batch_size != 2048 + or batch_size != 2048 + or validate is not False + or shared_cache + ): + raise TypeError("Parquet conversion options require a Parquet source") if urlpath is None: raise TypeError("RemoteCTable requires a remote B2Z URL") settings = { @@ -136,6 +209,15 @@ def __init__(self, *args, **kwargs): # Construction is completed by CTable._open_from_storage() in __new__. pass + @classmethod + def open_reference(cls, path, *, storage_options=None, parquet_options=None): + """Reopen a saved Parquet table with replacement runtime options.""" + from blosc2.remote_parquet import RemoteParquetCTable + + return RemoteParquetCTable.open_reference( + path, storage_options=storage_options, parquet_options=parquet_options + ) + @classmethod def with_sparse_cache( cls, @@ -165,6 +247,37 @@ def with_sparse_cache( ``max_cache_bytes=None`` for unlimited retention. For ordinary shared caching, prefer ``blosc2.open(url, cache_dir=..., shared_cache=True)``. """ + if isinstance(urlpath, (str, os.PathLike)) and os.fspath(urlpath).split("?", 1)[0].lower().endswith( + ".parquet" + ): + if any( + value is not None + for value in ( + dataset, + path, + manifest, + carrier, + _filesystem, + _filesystem_resolver, + _batch_validator, + _source_validator, + _manifest_validator, + _max_nodes, + ) + ): + raise ValueError("Parquet sparse caching accepts a single file and cache directory") + from blosc2.remote_parquet import RemoteParquetCTable + + return RemoteParquetCTable( + os.fspath(urlpath), + cache_dir=runtime_cache_path, + shared_cache=True, + max_cache_bytes=max_cache_bytes, + storage_options=storage_options, + max_concurrency=max_concurrency, + metadata_buffer_bytes=metadata_buffer_bytes, + row_buffer_bytes=row_buffer_bytes, + ) settings = { name: _positive_integer(name, value) for name, value in { diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py new file mode 100644 index 000000000..ad233bf74 --- /dev/null +++ b/src/blosc2/remote_parquet.py @@ -0,0 +1,822 @@ +"""Read-only, row-group-backed Parquet tables.""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +import threading +import uuid +import zipfile +from collections import OrderedDict +from contextlib import contextmanager +from dataclasses import fields +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import numpy as np + +import blosc2 +from blosc2.ctable import CTable, NullPolicy, get_null_policy, null_policy +from blosc2.ctable_storage import RemoteTableStorage, _AllValidRows, split_field_path +from blosc2.proxy_source import Traffic +from blosc2.remote_array import CACHE_POLICY_DEFAULT, normalize_cache_limit +from blosc2.remote_ctable import RemoteCTable, _positive_integer +from blosc2.remote_store_cache import lock_cache_file +from blosc2.schema_compiler import schema_to_dict + + +def _portable(value): + if isinstance(value, np.generic): + value = value.item() + if value is None or isinstance(value, bool | int | float | str | bytes): + return value + if isinstance(value, (list, tuple)): + return [_portable(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {key: _portable(item) for key, item in value.items()} + raise TypeError(f"Nonportable Parquet reader or conversion option: {type(value).__name__}") + + +class _CountingHandle: + def __init__(self, handle, traffic): + self.handle = handle + self.traffic = traffic + + def read(self, size=-1): + data = self.handle.read(size) + self.traffic.charge(len(data)) + return data + + def readinto(self, buffer): + size = self.handle.readinto(buffer) + self.traffic.charge(size) + return size + + def __getattr__(self, name): + return getattr(self.handle, name) + + +def _source_marker(urlpath, storage_options): + import fsspec + + fs, path = fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + info = fs.info(path) + return { + key: str(info[key]) + for key in ( + "size", + "etag", + "ETag", + "version_id", + "mtime", + "created", + "LastModified", + "Last-Modified", + ) + if info.get(key) is not None + } + + +def _disk_cache_path(urlpath, storage_options, options, cache_dir, marker): + import pyarrow + + if "size" not in marker or len(marker) < 2: + raise ValueError("A persistent Parquet cache requires source size and an ETag or modification time") + identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, marker, options) + digest = hashlib.sha256(repr(identity).encode()).hexdigest() + directory = Path(cache_dir) / "parquet" / digest + directory.mkdir(parents=True, exist_ok=True) + return directory + + +@contextmanager +def _disk_guard(directory): + if directory is None: + yield + return + with (directory / "owner.lock").open("a+b") as lock: + lock_cache_file(lock, blocking=True) + yield + + +def _group_filename(number, physical): + token = hashlib.sha256(physical.encode()).hexdigest()[:16] + return f"{number}-{token}.b2z" + + +class _ParquetOwner: + format = "parquet" + is_mutable = True + shared = False + + def __init__(self, path, handle, parquet_file, options, *, cache_policy, max_cache_bytes, cache_dir): + self.urlpath = path + self.handle = handle + self.traffic = handle.traffic + self.parquet_file = parquet_file + self.options = options + self.lock = threading.RLock() + self.generation = 0 + self.users = 0 + self.closed = False + self.cache = OrderedDict() + self.cache_bytes = 0 + self.cache_policy = cache_policy + self.max_cache_bytes = max_cache_bytes + self.cache_dir = cache_dir + self._cache_manager = None + self.row_ends = np.cumsum( + [parquet_file.metadata.row_group(i).num_rows for i in range(parquet_file.num_row_groups)] + ) + + def acquire(self): + self.users += 1 + + def release(self): + self.users -= 1 + if self.users == 0: + self.closed = True + self.cache.clear() + self.parquet_file.close() + self.handle.close() + if self._cache_manager is not None: + self._cache_manager.cleanup() + + def group(self, number, physical): + key = number, physical + with self.lock: + with _disk_guard(self.cache_dir): + return self._group_locked(number, physical, key) + + def _group_locked(self, number, physical, key): + if key in self.cache: + self.cache.move_to_end(key) + return self.cache[key][0] + cache_file = None + if self.cache_dir is not None: + cache_file = self.cache_dir / _group_filename(number, physical) + if cache_file.exists(): + try: + cached = CTable.load(cache_file) + start = 0 if number == 0 else int(self.row_ends[number - 1]) + if len(cached) == int(self.row_ends[number]) - start: + return cached + cached.close() + except (OSError, ValueError, RuntimeError, KeyError, TypeError, zipfile.BadZipFile): + pass # Interrupted or corrupt publication: rebuild this group. + arrow = self.parquet_file.read_row_group(number, columns=[physical], use_threads=False) + if physical == "" and not self.options["flatten_root"]: + arrow = arrow.rename_columns([self.options["root_name"]]) + meta = dict(arrow.schema.metadata or {}) + meta[b"blosc2_empty_root_physical"] = self.options["root_name"].encode() + arrow = arrow.replace_schema_metadata(meta) + with null_policy(self.options["null_policy"]): + table = CTable.from_arrow( + arrow.schema, + arrow.to_batches(max_chunksize=self.options["batch_size"]), + string_max_length=self.options["string_max_length"], + auto_null_sentinels=self.options["auto_null_sentinels"], + null_storage=self.options["null_storage"], + separate_nested_cols=self.options["separate_nested_cols"], + list_serializer=self.options["list_serializer"], + blosc2_batch_size=self.options["blosc2_batch_size"], + blosc2_items_per_block=self.options["blosc2_items_per_block"], + cparams=self.options["cparams"], + dparams=self.options["dparams"], + validate=self.options["validate"], + ) + size = int(getattr(table, "cbytes", 0) or arrow.nbytes) + if cache_file is not None: + temporary = cache_file.with_name(f".{uuid.uuid4().hex}.b2z") + try: + with table.copy(urlpath=temporary): + pass + os.replace(temporary, cache_file) + finally: + temporary.unlink(missing_ok=True) + self._evict_disk(cache_file) + elif self.cache_policy is blosc2.CachePolicy.MEMORY: + self.cache[key] = table, size + self.cache_bytes += size + while ( + self.max_cache_bytes is not None + and self.cache_bytes > self.max_cache_bytes + and len(self.cache) > 1 + ): + _, (_, removed) = self.cache.popitem(last=False) + self.cache_bytes -= removed + return table + + def _evict_disk(self, newest): + if self.max_cache_bytes is None: + return + entries = sorted(self.cache_dir.glob("*.b2z"), key=lambda path: path.stat().st_mtime_ns) + total = sum(path.stat().st_size for path in entries) + total += sum(path.stat().st_size for path in self.cache_dir.glob("row-map-*.npy")) + for path in entries: + if total <= self.max_cache_bytes: + break + if path != newest: + size = path.stat().st_size + path.unlink(missing_ok=True) + total -= size + + +class _ParquetColumn: + def __init__(self, storage, name, mask=False): + self.storage = storage + self.name = name + self.mask = mask + self.shape = (storage.length,) + self.chunks = (min(max(storage.length, 1), 65536),) + self.blocks = self.chunks + self.dtype = np.dtype(bool) if mask else storage.schema.columns_by_name[name].dtype + self._dictionary = None + + def __len__(self): + return self.shape[0] + + def flush(self): + pass + + @property + def dictionary(self): + if self._dictionary is not None: + return self._dictionary + values = [] + seen = set() + for group in range(len(self.storage.row_ends)): + table = self.storage._owner.group(group, self.storage.physical[self.name]) + try: + for value in table._cols[self.name].dictionary: + if value not in seen: + values.append(value) + seen.add(value) + finally: + if self.storage._owner.cache_policy is not blosc2.CachePolicy.MEMORY: + table.close() + self._dictionary = values + return values + + @property + def codes(self): + column = self + + class Codes: + def __getitem__(self, key): + mapping = {value: index for index, value in enumerate(column.dictionary)} + return np.asarray([mapping.get(value, -1) for value in column[key]], dtype=np.int32) + + return Codes() + + def __getitem__(self, key): + self.storage._check_open() + if isinstance(key, tuple) and len(key) == 1: + key = key[0] + scalar = isinstance(key, (int, np.integer)) + positions = np.arange(*key.indices(self.shape[0])) if isinstance(key, slice) else np.asarray(key) + if scalar: + positions = np.asarray([int(key)]) + elif positions.dtype == np.dtype(bool): + if positions.ndim != 1 or len(positions) != self.shape[0]: + raise IndexError("boolean row mask has the wrong length") + positions = np.flatnonzero(positions) + positions = np.asarray(positions, dtype=np.int64).reshape(-1) + positions = np.where(positions < 0, positions + self.shape[0], positions) + if np.any((positions < 0) | (positions >= self.shape[0])): + raise IndexError("row index out of range") + values = [None] * len(positions) + groups = np.searchsorted(self.storage.row_ends, positions, side="right") + for group in np.unique(groups): + selected = np.flatnonzero(groups == group) + start = 0 if group == 0 else int(self.storage.row_ends[group - 1]) + local = positions[selected] - start + table = self.storage._owner.group(int(group), self.storage.physical[self.name]) + try: + if self.mask: + mask = table._null_mask(self.name) + part = np.ones(len(local), dtype=bool) if mask is None else mask[local] + else: + part = table._cols[self.name][local] + finally: + if self.storage._owner.cache_policy is not blosc2.CachePolicy.MEMORY: + table.close() + for target, value in zip(selected, part, strict=True): + values[int(target)] = value + result = ( + np.asarray(values, dtype=self.dtype) + if self.dtype is not None and self.dtype != np.dtype(object) + else np.asarray(values, dtype=object) + ) + return result[0] if scalar else result + + +class ParquetTableStorage(RemoteTableStorage): + def __init__( + self, owner, schema, physical, length, *, max_concurrency, metadata_buffer_bytes, row_buffer_bytes + ): + self._owner = owner + self._generation = owner.generation + self._closed = False + self._root_key = "" + self.max_concurrency = max_concurrency + self.metadata_buffer_bytes = metadata_buffer_bytes + self.row_buffer_bytes = row_buffer_bytes + self.schema = schema + self.physical = physical + self.length = length + self.row_ends = owner.row_ends + owner.acquire() + + def check_kind(self): + self._check_open() + + def _metadata(self): + return {"kind": "ctable"} + + def load_user_attrs(self): + return {} + + def load_schema(self): + result = schema_to_dict(self.schema) + result["n_rows"] = self.length + return result + + def open_valid_rows(self): + return _AllValidRows(self.length, (65536,)) + + def open_columns(self, table, names, load): + for name in names: + load(name) + + def open_column(self, name): + return _ParquetColumn(self, name) + + open_list_column = open_column + + def open_dictionary_column(self, name, spec): + return self.open_column(name) + + def open_varlen_scalar_column(self, name, spec): + return self.open_column(name) + + def has_null_mask(self, name): + return bool(getattr(self.schema.columns_by_name[name].spec, "uses_mask", False)) + + def open_null_mask(self, name): + return _ParquetColumn(self, name, mask=True) + + def close(self): + if not self._closed: + self._closed = True + self._owner.release() + + +class RemoteParquetCTable(RemoteCTable): + """A read-only Parquet CTable that converts accessed row groups on demand.""" + + def __new__( # noqa: C901 + cls, + urlpath, + *, + storage_options=None, + columns=None, + max_rows=None, + parquet_options=None, + string_max_length=None, + auto_null_sentinels=True, + null_storage=None, + separate_nested_cols=True, + list_serializer="msgpack", + blosc2_batch_size=2048, + blosc2_items_per_block=None, + batch_size=2048, + cparams=None, + dparams=None, + validate=False, + max_cache_bytes=CACHE_POLICY_DEFAULT, + cache_policy=CACHE_POLICY_DEFAULT, + shared_cache=False, + max_concurrency=8, + metadata_buffer_bytes=8 << 20, + row_buffer_bytes=64 << 20, + _effective_null_policy=None, + **kwargs, + ): + import fsspec + import pyarrow as pa + import pyarrow.parquet as pq + + cache_dir = kwargs.pop("cache_dir", None) + seed_cache = kwargs.pop("_seed_cache", None) + cache_manager = kwargs.pop("_cache_manager", None) + if kwargs: + raise TypeError(f"Unsupported Parquet options: {', '.join(kwargs)}") + if max_rows is not None and max_rows < 0: + raise ValueError("max_rows must be non-negative") + CTable._validate_arrow_batch_size(batch_size) + settings = { + name: _positive_integer(name, value) + for name, value in { + "max_concurrency": max_concurrency, + "metadata_buffer_bytes": metadata_buffer_bytes, + "row_buffer_bytes": row_buffer_bytes, + }.items() + } + if columns is not None and len(set(columns)) != len(columns): + raise ValueError("columns must be unique") + if parquet_options and parquet_options.get("memory_map"): + raise ValueError("memory_map is incompatible with a remote Parquet handle") + if cache_policy is CACHE_POLICY_DEFAULT: + cache_policy = blosc2.CachePolicy.DISK if cache_dir is not None else blosc2.CachePolicy.MEMORY + else: + cache_policy = blosc2.CachePolicy(cache_policy) + if (cache_policy is blosc2.CachePolicy.DISK) != (cache_dir is not None): + raise ValueError("Parquet DISK cache policy requires cache_dir, and cache_dir requires DISK") + if shared_cache and cache_policy is not blosc2.CachePolicy.DISK: + raise ValueError("shared_cache=True requires a disk cache") + max_cache_bytes = normalize_cache_limit(cache_policy, max_cache_bytes) + handle = _CountingHandle(fsspec.open(urlpath, "rb", **(storage_options or {})).open(), Traffic()) + try: + pf = pq.ParquetFile(handle, **(parquet_options or {})) + fields = pf.schema_arrow + if columns is not None: + fields = pa.schema([fields.field(name) for name in columns]) + root_name = "root" + if "" in fields.names: + while root_name in fields.names: + root_name += "_1" + flatten_root = separate_nested_cols and CTable._detect_unnamed_root_list_struct(pa, fields) + effective_policy = _effective_null_policy or get_null_policy() + # Mask-backed schemas are determined by the Arrow schema alone. A + # value sample is only needed when in-band null handling may reject + # actual nulls during inference. + sample_needed = (null_storage or effective_policy.resolve_null_storage()) != "mask" + sample = ( + next(pf.iter_batches(batch_size=1 if flatten_root else batch_size, columns=columns), None) + if sample_needed + else None + ) + if sample is None: + sample = pa.RecordBatch.from_arrays( + [pa.array([], type=f.type) for f in fields], schema=fields + ) + if not flatten_root and "" in fields.names: + names = [root_name if name == "" else name for name in fields.names] + sample = sample.rename_columns(names) + meta = dict(sample.schema.metadata or {}) + meta[b"blosc2_empty_root_physical"] = root_name.encode() + sample = sample.replace_schema_metadata(meta) + options = { + "root_name": root_name, + "flatten_root": flatten_root, + "null_policy": effective_policy, + "string_max_length": string_max_length, + "auto_null_sentinels": auto_null_sentinels, + "null_storage": null_storage, + "separate_nested_cols": separate_nested_cols, + "list_serializer": list_serializer, + "blosc2_batch_size": blosc2_batch_size, + "blosc2_items_per_block": blosc2_items_per_block, + "batch_size": batch_size, + "cparams": cparams, + "dparams": dparams, + "validate": validate, + } + # A plain in-memory read needs only the file handle's metadata. + # Fetch a stronger source marker when disk caching or saving a + # portable reference actually needs one. + marker = _source_marker(urlpath, storage_options) if cache_dir is not None else None + disk_dir = ( + _disk_cache_path(urlpath, storage_options, options, cache_dir, marker) + if cache_dir is not None + else None + ) + if seed_cache: + if disk_dir is None: + raise ValueError("A retained Parquet cache requires cache_dir") + with _disk_guard(disk_dir): + for name, data in seed_cache.items(): + if ( + not isinstance(name, str) + or not isinstance(data, bytes) + or (not name.startswith("row-map-") and not name.endswith(".b2z")) + or Path(name).name != name + ): + raise ValueError("Invalid retained Parquet cache entry") + destination = disk_dir / name + with tempfile.NamedTemporaryFile(dir=disk_dir, delete=False) as staged: + staged.write(data) + try: + os.replace(staged.name, destination) + finally: + Path(staged.name).unlink(missing_ok=True) + with null_policy(options["null_policy"]): + probe = CTable.from_arrow( + sample.schema, + [sample], + string_max_length=string_max_length, + auto_null_sentinels=auto_null_sentinels, + null_storage=null_storage, + separate_nested_cols=separate_nested_cols, + list_serializer=list_serializer, + blosc2_batch_size=blosc2_batch_size, + blosc2_items_per_block=blosc2_items_per_block, + cparams=cparams, + dparams=dparams, + validate=validate, + ) + physical = { + name: ( + "" + if flatten_root or (split_field_path(name)[0] == root_name and "" in fields.names) + else name + if name in fields.names + else split_field_path(name)[0] + ) + for name in probe.col_names + } + if flatten_root: + # ponytail: full length scan; replace with a persisted offsets map when large nested roots matter. + with _disk_guard(disk_dir): + lengths = [] + count = 0 + map_file = disk_dir / f"row-map-{max_rows}.npy" if disk_dir is not None else None + ends = None + if map_file is not None and map_file.exists(): + try: + if map_file.stat().st_size <= 1024 + 8 * pf.num_row_groups: + loaded = np.load(map_file, allow_pickle=False) + if ( + loaded.ndim == 1 + and loaded.dtype == np.dtype(np.int64) + and len(loaded) <= pf.num_row_groups + and np.all(loaded >= 0) + and np.all(loaded[1:] >= loaded[:-1]) + and ( + len(loaded) == pf.num_row_groups + or max_rows == 0 + or (max_rows is not None and len(loaded) and loaded[-1] >= max_rows) + ) + ): + ends = loaded + except (OSError, ValueError): + pass + if ends is None: + for group in range(pf.num_row_groups): + if max_rows is not None and count >= max_rows: + break + row_group = pf.metadata.row_group(group) + leaf = min( + (row_group.column(i) for i in range(row_group.num_columns)), + key=lambda column: column.total_compressed_size, + ) + arr = pf.read_row_group(group, columns=[leaf.path_in_schema]).column(0) + size = pa.compute.sum(pa.compute.list_value_length(arr)).as_py() or 0 + lengths.append(size) + count += size + ends = np.cumsum(lengths) + if map_file is not None: + temporary = map_file.with_name(f".{uuid.uuid4().hex}.npy") + try: + np.save(temporary, ends) + os.replace(temporary, map_file) + finally: + temporary.unlink(missing_ok=True) + else: + ends = np.cumsum([pf.metadata.row_group(i).num_rows for i in range(pf.num_row_groups)]) + length = int(ends[-1]) if len(ends) else 0 + if max_rows is not None: + length = min(length, max_rows) + owner = _ParquetOwner( + urlpath, + handle, + pf, + options, + cache_policy=cache_policy, + max_cache_bytes=max_cache_bytes, + cache_dir=disk_dir, + ) + owner.source_marker = marker + owner._cache_manager = cache_manager + owner.reopen_kwargs = { + "storage_options": storage_options, + "columns": columns, + "max_rows": max_rows, + "parquet_options": parquet_options, + "string_max_length": string_max_length, + "auto_null_sentinels": auto_null_sentinels, + "null_storage": null_storage, + "separate_nested_cols": separate_nested_cols, + "list_serializer": list_serializer, + "blosc2_batch_size": blosc2_batch_size, + "blosc2_items_per_block": blosc2_items_per_block, + "batch_size": batch_size, + "cparams": cparams, + "dparams": dparams, + "validate": validate, + "max_cache_bytes": CACHE_POLICY_DEFAULT + if cache_policy is blosc2.CachePolicy.NONE + else max_cache_bytes, + "cache_policy": cache_policy, + "shared_cache": shared_cache, + "cache_dir": cache_dir, + "_effective_null_policy": effective_policy, + **settings, + } + owner.row_ends = ends + storage = ParquetTableStorage(owner, probe._schema, physical, length, **settings) + probe.close() + try: + return cls._open_from_storage(storage) + except BaseException: + storage.close() + raise + except BaseException: + handle.close() + raise + + def refresh(self): + storage = self._remote_storage() + owner = storage._owner + with owner.lock: + fresh = type(self)(owner.urlpath, **owner.reopen_kwargs) + fresh._remote_storage()._owner._cache_manager = owner._cache_manager + owner._cache_manager = None + owner.generation += 1 + state = fresh.__dict__.copy() + fresh._storage = None + self.__dict__ = state + self._cols._table = self + storage.close() + + @property + def source(self): + url = urlsplit(self._remote_storage()._owner.urlpath) + return { + "kind": "parquet", + "version": 1, + "urlpath": urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")), + } + + @property + def traffic(self): + return self._remote_storage()._owner.traffic + + @property + def cache_bytes(self): + owner = self._remote_storage()._owner + if owner.cache_dir is not None: + return sum( + path.stat().st_size for path in owner.cache_dir.iterdir() if path.suffix in {".b2z", ".npy"} + ) + return owner.cache_bytes + + @property + def metadata_bytes(self): + return self._remote_storage()._owner.parquet_file.metadata.serialized_size + + @property + def mutable(self): + return False + + @mutable.setter + def mutable(self, value): + raise RuntimeError("Parquet remote references are not supported") + + @property + def is_cache_mutable(self): + return True + + def save(self, *args, **kwargs): + return self._save_reference(*args, **kwargs) + + def _save_reference(self, destination, *, include_cache=True, overwrite=False, mutable=None): + """Write a CFrame-backed source reference without runtime credentials.""" + from blosc2.b2objects import make_b2object_carrier, write_b2object_payload + from blosc2.remote_array import validate_persistable_url + + if mutable not in (None, False): + raise ValueError("Parquet references are read-only") + owner = self._remote_storage()._owner + url = owner.urlpath + if urlsplit(url).scheme: + validate_persistable_url(url) + if owner.source_marker is None: + owner.source_marker = _source_marker(url, owner.reopen_kwargs["storage_options"]) + if "size" not in owner.source_marker or len(owner.source_marker) < 2: + raise ValueError("A portable Parquet reference requires an ETag or modification time") + options = { + key: value + for key, value in owner.reopen_kwargs.items() + if key + not in { + "storage_options", + "cache_dir", + "cache_policy", + "max_cache_bytes", + "shared_cache", + "_effective_null_policy", + } + and key not in {"max_concurrency", "metadata_buffer_bytes", "row_buffer_bytes"} + } + options["null_policy"] = { + field.name: getattr(owner.options["null_policy"], field.name) for field in fields(NullPolicy) + } + options = _portable(options) + payload = { + "kind": "remote_parquet", + "version": 1, + "urlpath": url, + "source_marker": owner.source_marker, + "options": options, + } + if include_cache: + retained = {} + with owner.lock, _disk_guard(owner.cache_dir): + if owner.cache_dir is not None: + retained.update( + (path.name, path.read_bytes()) + for path in owner.cache_dir.iterdir() + if path.suffix in {".b2z", ".npy"} + ) + else: + with tempfile.TemporaryDirectory() as cache_export: + for (number, physical), (table, _) in owner.cache.items(): + path = Path(cache_export) / _group_filename(number, physical) + with table.copy(urlpath=path): + pass + retained[path.name] = path.read_bytes() + payload["cache_files"] = retained + destination = Path(destination) + if destination.exists() and not overwrite: + raise FileExistsError(destination) + carrier = make_b2object_carrier("remote_parquet", (1,), np.uint8) + write_b2object_payload(carrier, payload) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as temporary: + staged = Path(temporary) / "reference.b2nd" + carrier.save(staged) + os.replace(staged, destination) + return str(destination) + + @classmethod + def _from_payload(cls, payload, *, storage_options=None, parquet_options=None): + from blosc2.remote_array import validate_persistable_url + + if ( + not isinstance(payload, dict) + or payload.get("kind") != "remote_parquet" + or payload.get("version") != 1 + ): + raise ValueError("Invalid Parquet reference") + url = payload.get("urlpath") + marker = payload.get("source_marker") + options = payload.get("options") + if not isinstance(url, str) or not isinstance(marker, dict) or not isinstance(options, dict): + raise ValueError("Invalid Parquet reference fields") + if urlsplit(url).scheme: + validate_persistable_url(url) + if "storage_options" in options or "cache_dir" in options: + raise ValueError("Parquet references cannot contain runtime transport options") + if _source_marker(url, storage_options) != marker: + raise RuntimeError("Parquet reference source has changed; create a new reference") + options = dict(options) + policy = options.pop("null_policy", None) + if not isinstance(policy, dict): + raise ValueError("Parquet reference has no null policy") + saved_reader = options.get("parquet_options") or {} + if parquet_options: + for key, value in parquet_options.items(): + if key in saved_reader and saved_reader[key] != value: + raise ValueError(f"Parquet reader option {key!r} differs from the saved reference") + options["parquet_options"] = {**saved_reader, **parquet_options} + retained = payload.get("cache_files", {}) + if not isinstance(retained, dict): + raise ValueError("Invalid retained Parquet cache") + manager = tempfile.TemporaryDirectory() if retained else None + try: + return cls( + url, + storage_options=storage_options, + _effective_null_policy=NullPolicy(**policy), + cache_dir=manager.name if manager is not None else None, + _seed_cache=retained, + _cache_manager=manager, + **options, + ) + except BaseException: + if manager is not None: + manager.cleanup() + raise + + @classmethod + def open_reference(cls, path, *, storage_options=None, parquet_options=None): + """Reopen a saved reference with replacement runtime transport options.""" + from blosc2.b2objects import read_b2object_payload + + raw = blosc2.blosc2_ext.open(os.fspath(path), "r", 0) + return cls._from_payload( + read_b2object_payload(raw), storage_options=storage_options, parquet_options=parquet_options + ) diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 27a087368..768ba09b0 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2687,7 +2687,7 @@ def _validate_shared_cache_request(urlpath, shared_cache, kwargs): kwargs["lazy"] = True -def open( +def open( # noqa: C901 urlpath: str | pathlib.Path | blosc2.URLPath, mode: str = "r", offset: int = 0, @@ -2857,12 +2857,15 @@ def open( Pre-computed native HDF5 index, or a local path or remote fsspec URL to its JSON encoding. It must match the source HDF5 URL and dataset scope. Arrays, PyTables tables, and hierarchy stores are supported. - source_format: {None, "blosc2", "zarr", "hdf5", "b2z"}, optional + source_format: {None, "blosc2", "zarr", "hdf5", "b2z", "parquet"}, optional Format of a lazy remote source. A ``.zarr`` URL path component selects Zarr automatically; a ``.h5`` or ``.hdf5`` path selects HDF5 automatically; a ``.b2z`` path selects B2Z automatically. An explicit value supports - suffix-free paths. Zarr, HDF5 and B2Z sources automatically enable + suffix-free paths. Zarr, HDF5, B2Z and Parquet sources automatically enable ``lazy=True``. + parquet_options: dict, optional + PyArrow ``ParquetFile`` reader options for a Parquet source. Conversion + options such as ``columns`` and ``max_rows`` are passed separately. assume_immutable: bool, optional With ``lazy=True``, skip remote identity checks before reads. Defaults to ``True``. Local disk caches always assume immutable sources; set to a new path or @@ -2964,6 +2967,40 @@ def open( True """ dataset = blosc2.core.resolve_dataset_path(dataset, path) + if kwargs.get("source_format") == "parquet" or ( + isinstance(urlpath, (str, pathlib.Path)) + and str(urlpath).split("?", 1)[0].lower().endswith(".parquet") + ): + if kwargs.get("source_format") not in (None, "parquet"): + raise ValueError("source_format conflicts with the .parquet suffix") + if mode != "r" or offset or dataset is not None: + raise ValueError("Parquet sources require mode='r', offset=0, and no dataset path") + from blosc2.remote_parquet import RemoteParquetCTable + + kwargs.pop("source_format", None) + if shared_cache: + if kwargs.get("cache_dir") is None: + raise ValueError("shared_cache=True requires cache_dir") + kwargs["shared_cache"] = True + lazy = kwargs.pop("lazy", True) + if lazy is False: + from blosc2.core import fsspec_open + + cache_keys = kwargs.keys() & { + "cache_dir", + "cache_policy", + "max_cache_bytes", + "max_concurrency", + "metadata_buffer_bytes", + "row_buffer_bytes", + } + if cache_keys: + raise TypeError(f"{', '.join(sorted(cache_keys))} require lazy Parquet access") + options = kwargs.pop("storage_options", None) + reader = kwargs.pop("parquet_options", None) or {} + with fsspec_open(str(urlpath), "rb", options) as handle: + return blosc2.CTable.from_parquet(handle, **kwargs, **reader) + return RemoteParquetCTable(str(urlpath), **kwargs) _reject_table_buffer_options(kwargs) _validate_shared_cache_request(urlpath, shared_cache, kwargs) if isinstance(urlpath, blosc2.URLPath): diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py new file mode 100644 index 000000000..af74cc1df --- /dev/null +++ b/tests/ctable/test_remote_parquet.py @@ -0,0 +1,649 @@ +"""Parity and row-group isolation for the Parquet remote-table prototype.""" + +import http.server +import os +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from decimal import Decimal +from email.utils import formatdate + +import fsspec +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +import blosc2 +from blosc2.schema_compiler import schema_to_dict + + +@pytest.mark.parametrize( + "source", + [ + pa.table({"id": [1, 2, None, 4], "text": ["a", None, "é", ""]}), + pa.table( + { + "unsigned": pa.array([0, 1, None, 255], type=pa.uint8()), + "float": pa.array([1.0, float("nan"), None, 4.5], type=pa.float32()), + "flag": pa.array([True, None, False, True], type=pa.bool_()), + } + ), + pa.table( + { + "large_text": pa.array(["é", "", None, "xyz"], type=pa.large_string()), + "binary": pa.array([b"\x00x", b"", None, b"hi"], type=pa.large_binary()), + } + ), + pa.table({"time": pa.array([0, None, 1234, 5678], type=pa.timestamp("ms", tz="UTC"))}), + pa.table({"items": pa.array([[1, None], [], None, [4]], type=pa.list_(pa.int64()))}), + pa.table( + { + "record": pa.array( + [{"id": 1}, None, {"id": 3}, {"id": 4}], + type=pa.struct([("id", pa.int32())]), + ) + } + ), + pa.table({"category": pa.array(["a", "b", "a", "c"]).dictionary_encode()}), + pa.table({"cell": pa.array([[1, 2], [2, 3], [3, 4], [5, 6]], type=pa.list_(pa.int32(), 2))}), + pa.table( + { + "items": pa.array( + [[{"id": 1, "name": "a"}], None, [], [{"id": None, "name": "b"}]], + type=pa.list_(pa.struct([("id", pa.int32()), ("name", pa.string())])), + ) + } + ), + pa.table( + { + "trip": pa.array( + [ + {"begin": {"lon": 1.5, "when": 1000}}, + None, + {"begin": {"lon": None, "when": None}}, + {"begin": {"lon": 4.5, "when": 4000}}, + ], + type=pa.struct( + [("begin", pa.struct([("lon", pa.float64()), ("when", pa.timestamp("ms"))]))] + ), + ) + } + ), + pa.table({"id": pa.array([], type=pa.int32()), "text": pa.array([], type=pa.string())}), + pa.Table.from_arrays( + [pa.array([1, 2, 3, 4]), pa.array([5, 6, 7, 8]), pa.array([9, 10, 11, 12])], + names=["", "root", "foo.bar"], + ), + pa.Table.from_arrays( + [pa.array([[{"id": 1}], [], None, [{"id": 4}]], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ), + ], + ids=[ + "scalar", + "numeric-bool", + "large-string-binary", + "timestamp", + "list", + "struct", + "dictionary", + "fixed-cell", + "list-struct", + "nested-struct-timestamp", + "empty", + "escaped-root", + "flattened-root", + ], +) +def test_remote_parquet_matches_importer(source, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table(source, path, row_group_size=2) + fs = fsspec.filesystem("memory") + fs.pipe("/remote-parity.parquet", path.read_bytes()) + with blosc2.CTable.from_parquet(path) as eager, blosc2.open("memory:///remote-parity.parquet") as remote: + assert remote.col_names == eager.col_names + assert len(remote) == len(eager) + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + if "float" in source.column_names: + np.testing.assert_equal(remote.to_arrow().to_pylist(), eager.to_arrow().to_pylist()) + else: + assert remote.to_arrow().equals(eager.to_arrow()) + if "cell" not in source.column_names: + assert list(remote) == list(eager) + + +def test_narrow_read_uses_only_one_later_group(tmp_path): + path = tmp_path / "wide.parquet" + pq.write_table(pa.table({"one": list(range(12)), "two": list(range(12))}), path, row_group_size=3) + fs = fsspec.filesystem("memory") + fs.pipe("/remote-wide.parquet", path.read_bytes()) + with blosc2.open("memory:///remote-wide.parquet") as remote: + before = remote.traffic.requests + assert remote["one"][10] == 10 + assert remote.traffic.requests == before + 1 + assert remote["one"][10] == 10 + assert remote.traffic.requests == before + 1 + + +def test_unnamed_root_without_flattening(tmp_path): + path = tmp_path / "root.parquet" + source = pa.Table.from_arrays( + [pa.array([[{"id": 1}], [], None, [{"id": 4}]], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.CTable.from_parquet(path, separate_nested_cols=False) as eager: + with blosc2.open(path, separate_nested_cols=False) as remote: + assert len(remote) == len(eager) == 4 + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_empty_flattened_root(tmp_path): + path = tmp_path / "empty-root.parquet" + source = pa.Table.from_arrays( + [pa.array([[], None, []], type=pa.list_(pa.struct([("id", pa.int32())])))], names=[""] + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.CTable.from_parquet(path) as eager, blosc2.open(path) as remote: + assert len(remote) == len(eager) == 0 + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_flattened_root_counts_from_smallest_leaf(tmp_path): + path = tmp_path / "wide-root.parquet" + source = pa.Table.from_arrays( + [ + pa.array( + [ + [{"id": 1, "blob": os.urandom(128_000)}], + [], + [{"id": 2, "blob": os.urandom(128_000)}], + [{"id": 3, "blob": os.urandom(128_000)}], + ] + ) + ], + names=[""], + ) + pq.write_table(source, path, row_group_size=2, compression="NONE") + with blosc2.open(path) as remote: + assert len(remote) == 3 + assert remote.traffic.requests == 3 # Footer and one leaf per group. + assert remote.traffic.nbytes < path.stat().st_size // 2 + + +@pytest.mark.parametrize("limit", [0, 2, 5]) +def test_flattened_root_max_rows(tmp_path, limit): + path = tmp_path / "root.parquet" + source = pa.Table.from_arrays( + [ + pa.array( + [[{"id": 1}, {"id": 2}], [], None, [{"id": 4}], [{"id": 5}, {"id": 6}]], + type=pa.list_(pa.struct([("id", pa.int32())])), + ) + ], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.CTable.from_parquet(path, max_rows=limit) as eager: + with blosc2.open(path, max_rows=limit) as remote: + assert len(remote) == len(eager) + assert remote.to_arrow().equals(eager.to_arrow()) + + +@pytest.mark.parametrize("serializer", ["msgpack", "arrow"]) +def test_nested_list_serializers(tmp_path, serializer): + path = tmp_path / "nested.parquet" + source = pa.table( + {"items": pa.array([[[1, None], []], None, [], [[4]]], type=pa.list_(pa.list_(pa.int32())))} + ) + pq.write_table(source, path, row_group_size=2) + options = {"list_serializer": serializer, "blosc2_batch_size": 2, "blosc2_items_per_block": 1} + with blosc2.CTable.from_parquet(path, **options) as eager, blosc2.open(path, **options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_null_policy_frozen_at_open(tmp_path): + path = tmp_path / "nullable.parquet" + pq.write_table(pa.table({"id": pa.array([1, None, 3], type=pa.int8())}), path, row_group_size=1) + with blosc2.null_policy(blosc2.NullPolicy(signed_int_strategy="max")): + eager = blosc2.CTable.from_parquet(path) + remote = blosc2.open(path) + try: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + finally: + eager.close() + remote.close() + + +def test_selection_and_materialization(tmp_path): + path = tmp_path / "selected.parquet" + target = tmp_path / "selected.b2z" + pq.write_table(pa.table({"a": [0, 1, 2, 3], "b": [10, 11, 12, 13]}), path, row_group_size=2) + with blosc2.open(path, columns=["b", "a"], max_rows=3) as remote: + assert remote.col_names == ["b", "a"] + assert [row.a for row in remote] == [0, 1, 2] + assert remote.where("a > 0").to_arrow().column("a").to_pylist() == [1, 2] + with remote.copy(urlpath=target) as materialized: + assert materialized.to_arrow().equals(remote.to_arrow()) + with blosc2.open(target) as reopened: + assert reopened.to_arrow().column("b").to_pylist() == [10, 11, 12] + + +def test_direct_constructor_and_settings(tmp_path): + path = tmp_path / "direct.parquet" + pq.write_table(pa.table({"a": [1, 2]}), path) + with blosc2.RemoteCTable(path, max_concurrency=2, row_buffer_bytes=1024) as remote: + assert remote["a"][:].tolist() == [1, 2] + assert remote.max_concurrency == 2 + assert remote.row_buffer_bytes == 1024 + + +def test_explicit_extensionless_source_and_projection(tmp_path): + path = tmp_path / "source" + pq.write_table(pa.table({"a": [1, 2], "b": [3, 4]}), path) + with blosc2.open(path, source_format="parquet", columns=["b"]) as remote: + assert remote.col_names == ["b"] + assert remote.to_arrow().column("b").to_pylist() == [3, 4] + with pytest.raises(KeyError): + blosc2.open(path, source_format="parquet", columns=["missing"]) + + +def test_unsupported_type_rejected_at_open(tmp_path): + path = tmp_path / "decimal.parquet" + pq.write_table(pa.table({"amount": pa.array([Decimal("1.00")], type=pa.decimal128(6, 2))}), path) + with pytest.raises(TypeError, match="No blosc2 spec"): + blosc2.open(path) + + +def test_invalid_source_options_fail_clearly(tmp_path): + path = tmp_path / "data.parquet" + pq.write_table(pa.table({"a": [1]}), path) + with pytest.raises(ValueError, match="mode='r'"): + blosc2.open(path, mode="a") + with pytest.raises(TypeError, match="require lazy"): + blosc2.open(path, lazy=False, cache_dir=tmp_path / "cache") + with pytest.raises(ValueError, match="requires cache_dir"): + blosc2.open(path, shared_cache=True) + with pytest.raises(ValueError, match="memory_map"): + blosc2.open(path, parquet_options={"memory_map": True}) + with pytest.raises(ValueError, match="conflicts"): + blosc2.open(path, source_format="blosc2") + + +def test_portable_reference_reopens_and_detects_source_change(tmp_path): + path = tmp_path / "source.parquet" + carrier = tmp_path / "source.b2nd" + pq.write_table(pa.table({"id": pa.array([1, None, 3], type=pa.int8())}), path) + with blosc2.null_policy(blosc2.NullPolicy(signed_int_strategy="max")): + with blosc2.open(path) as remote: + remote.save(carrier) + with blosc2.open(carrier) as reopened: + assert reopened.to_arrow().column("id").to_pylist() == [1, None, 3] + with blosc2.RemoteCTable.open_reference(carrier) as reopened: + assert reopened.to_arrow().column("id").to_pylist() == [1, None, 3] + output = subprocess.check_output( + [ + sys.executable, + "-c", + "import blosc2,sys; print(blosc2.open(sys.argv[1]).to_arrow().column('id').to_pylist())", + str(carrier), + ], + text=True, + ) + assert output.strip() == "[1, None, 3]" + pq.write_table(pa.table({"id": pa.array([4, 5], type=pa.int8())}), path) + with pytest.raises(RuntimeError, match="source has changed"): + blosc2.open(carrier) + + +def test_reference_retains_cached_group_and_runtime_options(tmp_path): + path = tmp_path / "source.parquet" + carrier = tmp_path / "source.b2nd" + pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) + with blosc2.open(path, storage_options={"auto_mkdir": True}) as remote: + assert remote["id"][5] == 5 + remote.save(carrier, include_cache=True) + with blosc2.RemoteCTable.open_reference(carrier, storage_options={"auto_mkdir": True}) as reopened: + before = reopened.traffic.requests + assert reopened["id"][5] == 5 + assert reopened.traffic.requests == before + + +def test_reference_retains_flattened_row_map(tmp_path): + path = tmp_path / "root.parquet" + carrier = tmp_path / "root.b2nd" + source = pa.Table.from_arrays( + [pa.array([[{"id": i}] for i in range(6)], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.open(path, cache_dir=tmp_path / "cache") as remote: + assert remote["id"][5] == 5 + remote.save(carrier, include_cache=True) + with blosc2.open(carrier) as reopened: + assert len(reopened) == 6 + before = reopened.traffic.requests + assert reopened["id"][5] == 5 + assert reopened.traffic.requests == before + + +def test_reference_rejects_nonportable_reader_option(tmp_path): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"id": [1]}), path) + with blosc2.open(path) as remote: + remote._storage._owner.reopen_kwargs["parquet_options"] = {"decryption_properties": object()} + with pytest.raises(TypeError, match="Nonportable"): + remote.save(tmp_path / "source.b2nd") + + +def test_reference_reader_options_are_frozen(tmp_path): + path = tmp_path / "source.parquet" + carrier = tmp_path / "source.b2nd" + pq.write_table(pa.table({"name": ["a", "b", "a"]}), path) + with blosc2.open(path, parquet_options={"read_dictionary": ["name"]}) as remote: + remote.save(carrier, include_cache=False) + with blosc2.RemoteCTable.open_reference(carrier) as reopened: + assert reopened.to_arrow().column("name").to_pylist() == ["a", "b", "a"] + with pytest.raises(ValueError, match="differs"): + blosc2.RemoteCTable.open_reference(carrier, parquet_options={"read_dictionary": []}) + + +def test_closed_handle_rejects_cached_column(tmp_path): + path = tmp_path / "closed.parquet" + pq.write_table(pa.table({"a": [1, 2]}), path) + remote = blosc2.open(path) + column = remote._cols["a"] + remote.close() + with pytest.raises(RuntimeError, match="closed"): + column[0] + + +def test_refresh_invalidates_old_views(tmp_path): + path = tmp_path / "refresh.parquet" + pq.write_table(pa.table({"a": [1, 2]}), path) + with blosc2.open(path) as remote: + old_column = remote._cols["a"] + old_view = remote.select(["a"]) + pq.write_table(pa.table({"a": [3, 4]}), path) + remote.refresh() + assert remote["a"][:].tolist() == [3, 4] + with pytest.raises(RuntimeError, match=r"closed|stale"): + old_column[0] + with pytest.raises(RuntimeError, match=r"closed|stale"): + old_view.to_arrow() + + +def test_reader_options_and_eager_path(tmp_path): + path = tmp_path / "dictionary.parquet" + pq.write_table(pa.table({"name": ["aa", "bb", "aa", None]}), path, row_group_size=2) + options = {"read_dictionary": ["name"]} + with blosc2.CTable.from_parquet(path, **options) as eager: + with blosc2.open(path, parquet_options=options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + with blosc2.open(path, lazy=False, parquet_options=options) as imported: + assert imported.to_arrow().equals(eager.to_arrow()) + + +def test_timestamp_reader_coercion(tmp_path): + path = tmp_path / "int96.parquet" + source = pa.table({"time": pa.array([0, None, 1234], type=pa.timestamp("ms"))}) + pq.write_table(source, path, row_group_size=2, use_deprecated_int96_timestamps=True) + options = {"coerce_int96_timestamp_unit": "ms"} + with blosc2.CTable.from_parquet(path, **options) as eager: + with blosc2.open(path, parquet_options=options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_fixed_string_width_error_on_access(tmp_path): + path = tmp_path / "strings.parquet" + pq.write_table(pa.table({"s": ["a", "long"]}), path, row_group_size=1) + with blosc2.open(path, string_max_length=2) as remote: + assert remote["s"][0] == "a" + with pytest.raises(ValueError, match="longer than max_length=2"): + remote["s"][1] + + +def test_fixed_string_width_mapping_matches_eager(tmp_path): + path = tmp_path / "strings.parquet" + pq.write_table( + pa.table({"s": ["a", "é", None], "b": [b"\x00", b"hi", None], "v": ["wide", "text", ""]}), + path, + row_group_size=2, + ) + options = {"string_max_length": {"s": 2, "b": 2}} + with blosc2.CTable.from_parquet(path, **options) as eager, blosc2.open(path, **options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_reordered_duplicate_and_empty_rows(tmp_path): + path = tmp_path / "rows.parquet" + pq.write_table(pa.table({"x": [0, 1, 2, 3, 4]}), path, row_group_size=2) + with blosc2.open(path) as remote: + assert remote[-1].x == 4 + assert remote[[4, 1, 4]].to_arrow().column("x").to_pylist() == [4, 1, 4] + assert remote[2:0:-1].to_arrow().column("x").to_pylist() == [2, 1] + assert remote["x"][np.array([True, False, True, False, False])].tolist() == [0, 2] + assert len(remote[0:0]) == 0 + with pytest.raises(IndexError): + remote._cols["x"][5] + + +def test_dictionary_codes_change_between_groups(tmp_path): + path = tmp_path / "reordered-dictionary.parquet" + schema = pa.schema([pa.field("category", pa.dictionary(pa.int8(), pa.string()))]) + with pq.ParquetWriter(path, schema) as writer: + for dictionary in (["a", "b"], ["b", "a"]): + values = pa.DictionaryArray.from_arrays(pa.array([0, 1], type=pa.int8()), pa.array(dictionary)) + writer.write_batch(pa.record_batch([values], schema=schema)) + with blosc2.CTable.from_parquet(path) as eager, blosc2.open(path) as remote: + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_disk_cache_reuses_group_and_root_map(tmp_path): + path = tmp_path / "root.parquet" + source = pa.Table.from_arrays( + [pa.array([[{"id": i}] for i in range(6)], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + cache_dir = tmp_path / "cache" + with blosc2.open(path, cache_dir=cache_dir) as first: + prepared_requests = first.traffic.requests + assert first["id"][-1] == 5 + assert first.cache_bytes > 0 + with blosc2.open(path, cache_dir=cache_dir) as second: + reopened_requests = second.traffic.requests + assert reopened_requests < prepared_requests # row map reused + assert second["id"][-1] == 5 + assert second.traffic.requests == reopened_requests # converted group reused + + +def test_shared_disk_cache_reuses_converted_group_across_processes(tmp_path): + path = tmp_path / "shared.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir, shared_cache=True) as remote: + assert remote["id"][5] == 5 + script = ( + "import blosc2,sys; " + "t=blosc2.open(sys.argv[1],cache_dir=sys.argv[2],shared_cache=True); " + "before=t.traffic.requests; print(t['id'][5],t.traffic.requests-before); t.close()" + ) + output = subprocess.check_output([sys.executable, "-c", script, str(path), str(cache_dir)], text=True) + assert output.strip() == "5 0" + with blosc2.RemoteCTable.with_sparse_cache(path, cache_dir) as remote: + before = remote.traffic.requests + assert remote["id"][5] == 5 + assert remote.traffic.requests == before + + +def test_shared_disk_cache_concurrent_cold_reads(tmp_path): + path = tmp_path / "shared-cold.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": list(range(12))}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir, shared_cache=True) as first: + with blosc2.open(path, cache_dir=cache_dir, shared_cache=True) as second: + baseline = first.traffic.requests + second.traffic.requests + with ThreadPoolExecutor(max_workers=2) as pool: + assert list(pool.map(lambda table: table["id"][11], (first, second))) == [11, 11] + assert first.traffic.requests + second.traffic.requests == baseline + 1 + + +def test_disk_cache_eviction_and_corrupt_entry(tmp_path): + path = tmp_path / "data.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir, max_cache_bytes=1) as remote: + assert remote["id"][2] == 2 + assert remote["id"][5] == 5 + entries = list(cache_dir.rglob("*.b2z")) + assert len(entries) == 1 # one indivisible converted group exceeds the budget + entries[0].write_bytes(b"broken") + with blosc2.open(path, cache_dir=cache_dir) as remote: + assert remote["id"][5] == 5 + + +def test_refresh_disk_cache_uses_new_generation(tmp_path): + path = tmp_path / "changing.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": [1, 2]}), path) + with blosc2.open(path, cache_dir=cache_dir) as remote: + assert remote["id"][0] == 1 + pq.write_table(pa.table({"id": [3, 4, 5]}), path) + remote.refresh() + assert remote["id"][0] == 3 + + +def test_http_range_requests_are_narrow(tmp_path): + path = tmp_path / "served.parquet" + pq.write_table( + pa.table({"x": list(range(10_000)), "y": [f"value-{i:05d}" for i in range(10_000)]}), + path, + row_group_size=1_000, + ) + counts = {"requests": 0, "bytes": 0, "heads": 0} + + class Ranged(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + counts["heads"] += 1 + self.send_response(200) + self.send_header("Content-Length", str(path.stat().st_size)) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) + self.end_headers() + + def do_GET(self): + body = path.read_bytes() + span = self.headers.get("Range") + if span: + first, _, last = span.removeprefix("bytes=").partition("-") + if first: + first = int(first) + last = int(last) if last else len(body) - 1 + else: + first, last = max(0, len(body) - int(last)), len(body) - 1 + data = body[first : last + 1] + self.send_response(206) + self.send_header("Content-Range", f"bytes {first}-{last}/{len(body)}") + else: + data = body + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) + self.end_headers() + self.wfile.write(data) + counts["requests"] += 1 + counts["bytes"] += len(data) + + try: + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Ranged) + except PermissionError: + pytest.skip("localhost binding is blocked by the sandbox") + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}/served.parquet?signature=test" + with blosc2.open(url, storage_options={"block_size": 4096, "cache_type": "none"}) as remote: + assert counts["requests"] == 1 + assert counts["heads"] == 1 + assert "signature" not in remote.source["urlpath"] + with pytest.raises(ValueError, match="credential-like"): + remote.save(tmp_path / "signed.b2nd") + before = counts.copy() + assert remote["x"][9500] == 9500 + assert counts["requests"] > before["requests"] + assert counts["bytes"] - before["bytes"] < path.stat().st_size + warm = counts.copy() + assert remote["x"][9500] == 9500 + assert counts == warm + clean_url = url.split("?", 1)[0] + cache_dir = tmp_path / "http-cache" + with blosc2.open(clean_url, cache_dir=cache_dir) as remote: + assert remote["x"][9500] == 9500 + with blosc2.open(clean_url, cache_dir=cache_dir) as remote: + before = counts.copy() + assert remote["x"][9500] == 9500 + assert counts["requests"] == before["requests"] + finally: + server.shutdown() + server.server_close() + thread.join() + + +@pytest.mark.network +def test_s3_parquet_smoke(): + url = os.environ.get("BLOSC2_REMOTE_PARQUET_S3_URL") + if not url: + pytest.skip("set BLOSC2_REMOTE_PARQUET_S3_URL to an accessible Parquet file") + with blosc2.open(url, source_format="parquet") as remote: + assert remote.col_names + if len(remote): + remote[remote.col_names[0]][0] + + +def test_http_without_range_support_fails_clearly(tmp_path): + path = tmp_path / "large.parquet" + pq.write_table(pa.table({"x": list(range(20_000))}), path, row_group_size=1_000) + + class Unranged(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-Length", str(path.stat().st_size)) + self.end_headers() + + def do_GET(self): + body = path.read_bytes() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + try: + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Unranged) + except PermissionError: + pytest.skip("localhost binding is blocked by the sandbox") + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}/large.parquet" + with pytest.raises(ValueError, match="range requests"): + blosc2.open(url, storage_options={"block_size": 4096, "cache_type": "none"}) + finally: + server.shutdown() + server.server_close() + thread.join() From 7a376ae538ae536fe2c4004691bde13a58ca6eb9 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 11:08:44 +0200 Subject: [PATCH 02/19] Size remote Parquet row group conversions --- ...e_parquet_chicago_http_capacity_hint.jsonl | 5 +++ plans/remote-parquet.md | 32 ++++++++++++++++--- src/blosc2/remote_parquet.py | 1 + 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 bench/remote_parquet_chicago_http_capacity_hint.jsonl diff --git a/bench/remote_parquet_chicago_http_capacity_hint.jsonl b/bench/remote_parquet_chicago_http_capacity_hint.jsonl new file mode 100644 index 000000000..4aa2389bf --- /dev/null +++ b/bench/remote_parquet_chicago_http_capacity_hint.jsonl @@ -0,0 +1,5 @@ +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.203, "cold_reads": 1, "cold_bytes": 331590, "cold_heads": 0, "cold_seconds": 0.08, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32.parquet", "file_bytes": 686857703, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.021, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.083, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f64.parquet", "file_bytes": 687450036, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.023, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.085, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.066, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi.parquet", "file_bytes": 640173859, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 26, "open_bytes": 333139, "open_heads": 1, "open_seconds": 0.37, "cold_reads": 1, "cold_bytes": 6660483, "cold_heads": 0, "cold_seconds": 1.087, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.07, "cache_bytes": 12609432, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "company", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.205, "cold_reads": 1, "cold_bytes": 1831, "cold_heads": 0, "cold_seconds": 0.089, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.063, "cache_bytes": 1008, "warm_equal": true, "groups": 25, "arrow_equal": true} diff --git a/plans/remote-parquet.md b/plans/remote-parquet.md index dca684e80..a86fa1582 100644 --- a/plans/remote-parquet.md +++ b/plans/remote-parquet.md @@ -467,12 +467,11 @@ single sub-megabyte GET for `trip.sec`. For `chicago-taxi-flat-f32.parquet`, the final group has 259,680 rows and its `trip.sec` chunk is 352,939 compressed bytes. Reading and decoding that chunk directly with PyArrow took about 0.001 seconds from the local file. A `cProfile` -trace of the cold remote access showed 259,680 calls to `CTable._write` and +trace of the cold remote access showed 259,680 calls to `_ChunkAlignedWriter._write` and 259,712 calls to `NDArray.__setitem__` during `CTable.from_arrow()` conversion. Profiling raised -the total to 32.4 seconds versus 17.9 seconds uninstrumented. The dominant cost -is Python per-row materialization and index bookkeeping, not the HTTP transfer. -For faster random scalar access, conversion needs a vectorized path or a way to -read the selected value without materializing the entire row group. +the total to 32.4 seconds versus 17.9 seconds uninstrumented. Those calls came +from one-row Blosc2 chunks created without a capacity hint. Passing the known +row-group length fixes this bottleneck, as measured below. ### Request and transfer optimization (2026-09-25) @@ -516,6 +515,29 @@ The synthetic benchmark was rerun and saved in from 12 reads / 162,866 bytes to 11 reads / 109,198 bytes. Scalar, wide, and UTF-8 open counts stayed at one read. +### Row-group conversion capacity (2026-09-25) + +The remote reader now passes the known logical row-group length as +`capacity_hint` to `CTable.from_arrow()`. Without it, Arrow import sized the +flat column's initial Blosc2 chunks for one row, causing many small writes. +The localhost rerun is saved in +`bench/remote_parquet_chicago_http_capacity_hint.jsonl`; values still match +PyArrow. Request counts and transferred bytes are unchanged from the preceding +HTTP table. + +| File / column | Cold seconds before | Cold seconds after | Speedup | +| --- | ---: | ---: | ---: | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 17.571 | 0.080 | 220× | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 17.638 | 0.083 | 213× | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 17.862 | 0.085 | 210× | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 0.819 | 0.089 | 9.2× | +| `chicago-taxi.parquet` / `trip.sec` | 1.083 | 1.087 | 1.0× | + +Times are single local runs, including loopback transfer, conversion, and +value retrieval. The nested case already received a large logical capacity +through the Arrow importer's unnamed-root fallback. The flat converted-group +cache footprint also fell from about 34.7 MB to 356 KB for `trip.sec`. + - [ ] 1. Compatibility inventory, API contract, and storage seam - [ ] 2. Scalar lazy access - [ ] 3. Strings, binary, and dictionaries diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index ad233bf74..ef742fc84 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -185,6 +185,7 @@ def _group_locked(self, number, physical, key): cparams=self.options["cparams"], dparams=self.options["dparams"], validate=self.options["validate"], + capacity_hint=int(self.row_ends[number]) - (int(self.row_ends[number - 1]) if number else 0), ) size = int(getattr(table, "cbytes", 0) or arrow.nbytes) if cache_file is not None: From b8fd39a55a741dcce876949db1c749a2efc5acef Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 11:30:45 +0200 Subject: [PATCH 03/19] Project nested remote Parquet leaves --- ...parquet_chicago_http_leaf_projection.jsonl | 5 +++++ doc/guides/remote_tables.md | 3 ++- plans/remote-parquet.md | 17 +++++++++++++++ src/blosc2/remote_parquet.py | 21 +++++++++++++++++++ tests/ctable/test_remote_parquet.py | 11 +++++++++- 5 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 bench/remote_parquet_chicago_http_leaf_projection.jsonl diff --git a/bench/remote_parquet_chicago_http_leaf_projection.jsonl b/bench/remote_parquet_chicago_http_leaf_projection.jsonl new file mode 100644 index 000000000..72efc73da --- /dev/null +++ b/bench/remote_parquet_chicago_http_leaf_projection.jsonl @@ -0,0 +1,5 @@ +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.193, "cold_reads": 1, "cold_bytes": 331590, "cold_heads": 0, "cold_seconds": 0.081, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32.parquet", "file_bytes": 686857703, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.029, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.087, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.061, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f64.parquet", "file_bytes": 687450036, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.025, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.089, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.066, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi.parquet", "file_bytes": 640173859, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 26, "open_bytes": 333139, "open_heads": 1, "open_seconds": 0.355, "cold_reads": 1, "cold_bytes": 363880, "cold_heads": 0, "cold_seconds": 0.073, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.066, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "company", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.189, "cold_reads": 1, "cold_bytes": 1831, "cold_heads": 0, "cold_seconds": 0.094, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 1008, "warm_equal": true, "groups": 25, "arrow_equal": true} diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index 9489fe1c7..74d1db3a1 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -26,7 +26,8 @@ with blosc2.open( ``` Reads convert the requested physical field and row group using the Arrow -importer. A narrow scalar read can avoid unrelated fields and groups; a group +importer. Nested struct leaves are projected from Parquet when their physical +paths are unambiguous. A narrow scalar read can avoid unrelated fields and groups; a group with an indivisible large value can still allocate much more than the cache budget. With mask-backed nulls, schema inference reads only the footer; in-band null policies may need a first-batch sample. Flattening a single unnamed diff --git a/plans/remote-parquet.md b/plans/remote-parquet.md index a86fa1582..48fed2f0b 100644 --- a/plans/remote-parquet.md +++ b/plans/remote-parquet.md @@ -538,6 +538,23 @@ value retrieval. The nested case already received a large logical capacity through the Arrow importer's unnamed-root fallback. The flat converted-group cache footprint also fell from about 34.7 MB to 356 KB for `trip.sec`. +### Nested leaf projection (2026-09-25) + +The remote reader now maps a logical nested leaf to an unambiguous Parquet +column path using footer metadata. It falls back to the containing field when +the mapping is ambiguous. The converted cache is keyed by the projected path, +so sibling leaves do not force each other into the same conversion. Results are +saved in `bench/remote_parquet_chicago_http_leaf_projection.jsonl`. + +For `chicago-taxi.parquet / trip.sec`, opening still reads 26 GETs / 333,139 +bytes for the exact logical row map. The cold final-row read fell from one GET / +6,660,483 bytes / 1.087 s to one GET / 363,880 bytes / 0.073 s. The warm repeat +still makes no GET. Converted cache size fell from 12.6 MB to 356 KB. The other +four Chicago HTTP cases kept their request and transfer counts and had cold +times of 0.081–0.094 s. All measured final values match PyArrow. A local +comparison of the last nested row group also matched all 14 logical columns +against a full-field import, including the list-valued `trip.path` column. + - [ ] 1. Compatibility inventory, API contract, and storage seam - [ ] 2. Scalar lazy access - [ ] 3. Strings, binary, and dictionaries diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index ef742fc84..8d318b335 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -105,6 +105,20 @@ def _group_filename(number, physical): return f"{number}-{token}.b2z" +def _projected_field(name, physical, paths): + parts = split_field_path(name) + matches = set() + for path in paths: + segments = path.split(".") + for start in range(len(segments) - len(parts) + 1): + if tuple(segments[start : start + len(parts)]) != parts: + continue + prefix = ".".join(segments[: start + len(parts)]) + if not physical or prefix == physical or prefix.startswith(f"{physical}."): + matches.add(prefix) + return matches.pop() if len(matches) == 1 else physical + + class _ParquetOwner: format = "parquet" is_mutable = True @@ -538,6 +552,13 @@ def __new__( # noqa: C901 ) for name in probe.col_names } + paths = [pf.schema.column(i).path for i in range(len(pf.schema.names))] + physical = { + name: _projected_field(name, source, paths) + if flatten_root or (source and source != name) + else source + for name, source in physical.items() + } if flatten_root: # ponytail: full length scan; replace with a persisted offsets map when large nested roots matter. with _disk_guard(disk_dir): diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index af74cc1df..8039b6a89 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -130,7 +130,12 @@ def test_narrow_read_uses_only_one_later_group(tmp_path): def test_unnamed_root_without_flattening(tmp_path): path = tmp_path / "root.parquet" source = pa.Table.from_arrays( - [pa.array([[{"id": 1}], [], None, [{"id": 4}]], type=pa.list_(pa.struct([("id", pa.int32())])))], + [ + pa.array( + [[{"root": 1}], [], None, [{"root": 4}]], + type=pa.list_(pa.struct([("root", pa.int32())])), + ) + ], names=[""], ) pq.write_table(source, path, row_group_size=2) @@ -172,6 +177,10 @@ def test_flattened_root_counts_from_smallest_leaf(tmp_path): assert len(remote) == 3 assert remote.traffic.requests == 3 # Footer and one leaf per group. assert remote.traffic.nbytes < path.stat().st_size // 2 + before = remote.traffic.nbytes + assert remote["id"][-1] == 3 + assert remote.traffic.nbytes - before < path.stat().st_size // 2 + assert remote["blob"][-1] == source.column(0)[-1].as_py()[-1]["blob"] @pytest.mark.parametrize("limit", [0, 2, 5]) From 1024eebfd09eeb76cb75206aed57ef03d3b36425 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 12:43:28 +0200 Subject: [PATCH 04/19] Fix Parquet table info and plan cache unification --- plans/remote-parquet-cache-unification.md | 218 ++++++++++++++++++++++ src/blosc2/ctable_storage.py | 8 +- src/blosc2/remote_parquet.py | 10 + tests/ctable/test_remote_parquet.py | 12 ++ 4 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 plans/remote-parquet-cache-unification.md diff --git a/plans/remote-parquet-cache-unification.md b/plans/remote-parquet-cache-unification.md new file mode 100644 index 000000000..1b5cdad66 --- /dev/null +++ b/plans/remote-parquet-cache-unification.md @@ -0,0 +1,218 @@ +# Unify Parquet with the remote table cache lifecycle + +Proposal; no runtime changes implemented by this document. + +## Objective + +Treat a Parquet file as a table source managed by the existing RemoteStore / +RemoteCTable machinery. Use the same source naming, generation manifests, +ownership, cache budgets, refresh, direct cache reopening, and portable exports +as other remote table formats. Keep Parquet decoding and row mapping in its +source adapter. + +The unit fetched from Parquet can remain a physical field within a row group. +Consistency does not require mapping a Parquet row group to one Blosc2 chunk. + +## Current differences + +| Concern | Existing remote store machinery | Current Parquet implementation | +| --- | --- | --- | +| Disk identity | Readable source basename plus identity hash | `parquet/<64-character hash>` | +| Live container | `.b2d` and `active_generation.json` | Independent row-group `.b2z` files and optional `.npy` row maps | +| Ownership | `StoreDiskCache`; `SharedStoreCache` for shared operations | Separate `_disk_guard` used for disk reads | +| Cache budget | `CacheCoordinator` shared by retained payloads | Private memory LRU and disk eviction by modification time | +| Refresh | Prepare/publish generation, invalidate old handles | Reopen a separate Parquet owner and change its generation counter | +| Export | Remote-store manifest and retained payloads in a `.b2z` archive | `remote_parquet` CFrame reference containing cache files as byte strings | +| Reopening live cache | Root manifest reconstructs the remote table | Individual entries open as partial ordinary CTables | + +Relevant code: `remote_parquet.py`, `remote_store.py`, `remote_store_cache.py`, +`remote_ctable.py`, `ctable_storage.py`, and `proxy.py`. + +A terminology correction: `.b2d` is a directory container. It does not imply +that every array inside it uses `contiguous=False`. The common array code has +distinct ordinary contiguous carriers and shared sparse frames. Preserve that +distinction; do not turn every Parquet payload into a sparse frame merely to +obtain a `.b2d` root. + +## Proposed layout and identity + +```text +proves/ + chicago-taxi.parquet--/ + owner.lock + active_generation.json + .b2d/ + embed.b2e + _parquet/ + row-map.b2nd # Only when flattening requires a row map + groups/ + 0/ + .b2d/ + ...native CTable payloads... + 1/ + .b2d/ + ...native CTable payloads... +``` + +Use `StoreDiskCache.path_for()` and the existing `cache_directory_name()` / +path-component helpers. Do not implement another basename sanitizer or hashing +convention. The illustrated payload subtree is private; the public entry point +is the generation root, just as for other cached remote tables. + +The identity must include the normalized source location, access-configuration +fingerprint, projection, row limit, effective null policy, and options that +change decoded values or their cache representation. Keep credentials out of +persisted descriptors. Record cache format/conversion compatibility versions +explicitly. Persist the source revision marker in the generation metadata so +refresh replaces a generation within the same logical cache directory. + +Use a versioned Parquet section of the common manifest for the compiled schema, +physical/logical field mapping, row-group boundaries, decoding options, revision +marker, row map, and published group entries. Register the root as a `ctable` +node; auxiliary group tables are implementation details, not user-visible +tables in the logical hierarchy. + +## Reuse the lifecycle, retain a small Parquet adapter + +1. Add Parquet discovery and reconstruction to `RemoteDiscovery` and the common + source-descriptor validators. Route Parquet table construction through that + owner. Preserve the existing public `blosc2.open()` options and read-only + table behavior. Keeping `RemoteParquetCTable` as a thin compatibility class is + fine; it should no longer own a separate cache lifecycle. +2. Keep fsspec/PyArrow handles, footer decoding, logical row mapping, and + `CTable.from_arrow()` conversion in `remote_parquet.py`. Preserve current + nested/null/dictionary semantics and importer parity. +3. Make `ParquetTableStorage` obtain converted groups from the common owner. + Retain its column adapter for gathering values across unequal groups. The + existing common table storage assumes B2Z members in several paths, so simply + changing the owner class is insufficient: add explicit Parquet dispatch at + those storage boundaries. +4. Store each converted group/physical-field unit as native directory-backed + CTable payloads under the generation, replacing its ZIP wrapper. This avoids + requiring global string offsets or dictionary codes before all groups have + been visited. A physical field may supply multiple logical leaves and masks; + record that mapping rather than assuming one physical field is one scalar + column. +5. Register these units with `CacheCoordinator` through one small adapter, using + the existing remote batch cache as the model. The coordinator currently + expects concrete cache methods/maps; implement that interface, not a second + coordinator or a general plugin framework. + +This is substantive lifecycle reuse, not a `.b2d` wrapper around the current +standalone `.b2z` cache directory. Directly reopening the generation must expose +the complete logical table and fetch missing groups through its source adapter. + +## Reads, publication, accounting, and sharing + +A miss converts the required physical field for its row group, including any +dependent masks, dictionaries, and nested structure. Build it under a temporary +sibling directory. Under the common ownership lock, publish the completed unit +and then its manifest entry. A crash before publication must not turn a partial +unit into a cache hit. Recover abandoned staging/unreferenced payloads through +the same generation/dirty-state machinery used for other formats. + +Retain and evict complete converted units initially, including their dependent +payloads. This keeps dictionary and offset state coherent. Cache hits touch the +common LRU; reopened caches restore accounting from validated manifest entries. +Count retained compressed data consistently with the shared coordinator and +report metadata separately. Count directory/manifest overhead separately from +the compressed-payload budget, as with other stores. + +A row group larger than the budget must still be readable. Define the retained +state using the common budget contract: serve through a temporary live unit and +release it when it cannot be retained, rather than keeping an oversized group +indefinitely. Do not claim this bounds peak Arrow conversion memory. + +Ordinary DISK access uses the existing lifetime ownership lock. Shared access +uses `SharedStoreCache` and operation-scoped locking, with the normal manifest +reload, dirty recovery, and stale-handle checks. A shared cold miss must not +publish duplicate or inconsistent units. Complete group payloads can remain +immutable after publication; sparse per-chunk writes are unnecessary for them. + +NONE and MEMORY policies retain their documented behavior through the same +owner, without creating disk artifacts. Refresh prepares compatible metadata +and a new generation before retiring the old one. Failed refresh leaves the +previous table usable. Preserve source-change detection and require explicitly +defined validation when reopening a persisted generation; do not accidentally +weaken the current Parquet revision checks while adopting shared machinery. + +## Opening and transport + +These should be equivalent ways to recover the same logical remote table: + +```python +t = blosc2.open(url, cache_dir="proves") +t = blosc2.open("proves/chicago-taxi.parquet--/.b2d") +``` + +The generation root contains enough metadata to expose the whole schema and +reuse retained groups. Unknown sizes should be reported as unavailable or +explicitly partial; information display must not fetch all groups to compute +them. Opening and reads must honor common closed/stale-handle semantics. + +Use the existing remote-store `.save()` archive writer, extending payload +enumeration for Parquet group units and the row map. The new default portable +table artifact is a genuine `.b2z` remote-reference archive: + +```python +t.save("taxi-reference.b2z") # Include currently retained data +t.save("taxi-cold.b2z", include_cache=False) +u = blosc2.open("taxi-reference.b2z") +``` + +Export must not fetch unvisited groups. Stream/copy stored payloads into the +archive rather than embedding every file as an in-memory byte string in a +CFrame metadata payload. The shared archive reader must understand Parquet +segments and serve warm data from the exported file. Uncached reads still need +the source and runtime credentials. Cache mutability on reopening should follow +the existing remote-reference rules, independently of source read-only status. + +Local Parquet sources remain supported. Keep the distinction between reopening +a local cache and transporting a remote reference: a local source path does +not become available on another machine merely because its cache was exported. + +## Compatibility and rollout + +Implement and verify the internal integration before changing the disk layout. +Do not independently land a cosmetic rename that causes an extra cache rebuild. + +- Treat old `parquet/` runtime caches as disposable. Create the new cache + on the next open; leave old directories untouched and document cleanup. No + automatic migration is needed for these runtime files. +- Preserve a read-only compatibility loader for existing version-1 + `remote_parquet` CFrame references. Seed their retained data into the common + representation after validation. New `.save()` output uses the common archive + format; the loader must not retain a second runtime cache implementation. +- Version new manifest fields and validate paths, source revision, group + identity, shape, and conversion identity before accepting a cached unit. +- Keep current import options and numerical/null semantics. Do not combine this + change with a new Parquet page reader, global dictionary scheme, or per-chunk + group conversion strategy. + +## Implementation sequence and acceptance gates + +1. **Shared source ownership:** Add the Parquet table node and descriptor + metadata, reuse ordinary/shared ownership and generations, and preserve the + existing importer parity tests, including variable-length and nested data. +2. **Native payloads and common accounting:** Replace per-group ZIP files with + native group directories, integrate `CacheCoordinator`, and test publication, + eviction, reopened cache hits, and shared concurrent misses. +3. **Reopen/export integration:** Open the generation root as a full + `RemoteCTable`, export/import standard `.b2z` references, and support old saved + references through the compatibility loader. +4. **Remove duplicate paths and document:** Remove Parquet-specific path naming, + disk locking, LRU/eviction, lifecycle, and new-reference CFrame serialization. + Document the common layout and remaining Parquet fetch granularity. + +Completion requires tests covering unequal row groups, a read of another column +and another group, repeated warm reads with no payload refetch, persisted reopen, +complete logical schema at the generation root, refresh/stale views, aborted +publication, oversized units, and two processes sharing a cold miss. Export to +a different directory/process must preserve warm reads and allow cold misses to +fetch the source; old-reference loading must retain its source checks. Run the +existing remote-store/CTable cache and export tests alongside Parquet parity. + +Compare cold open, one narrow cold read, warm read, reopen, and export on a +multi-group mixed-type fixture. Record source traffic, retained bytes, and file +count. Changes must preserve column/group isolation and avoid repeated +conversion of retained data; no full-file import is an acceptable fallback. diff --git a/src/blosc2/ctable_storage.py b/src/blosc2/ctable_storage.py index 592a595db..f01bc4925 100644 --- a/src/blosc2/ctable_storage.py +++ b/src/blosc2/ctable_storage.py @@ -648,8 +648,12 @@ def index_anchor_path(self, col_name: str) -> str | None: def column_cbytes_for_info(column): - """Do not present shared HDF5 record storage as a per-field compressed size.""" - return None if isinstance(column, _RemoteHDF5Field) else getattr(column, "cbytes", None) + """Hide compressed sizes unavailable for remote fields.""" + return ( + None + if isinstance(column, _RemoteHDF5Field) or getattr(column, "_compressed_size_unavailable", False) + else getattr(column, "cbytes", None) + ) class _RemoteHDF5Field(blosc2.Operand): diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index 8d318b335..0c50f0b62 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -239,6 +239,8 @@ def _evict_disk(self, newest): class _ParquetColumn: + _compressed_size_unavailable = True + def __init__(self, storage, name, mask=False): self.storage = storage self.name = name @@ -252,6 +254,14 @@ def __init__(self, storage, name, mask=False): def __len__(self): return self.shape[0] + @property + def nbytes(self): + return len(self) * self.dtype.itemsize if self.dtype is not None else 0 + + @property + def cbytes(self): + return 0 + def flush(self): pass diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index 8039b6a89..376eabb6c 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -127,6 +127,18 @@ def test_narrow_read_uses_only_one_later_group(tmp_path): assert remote.traffic.requests == before + 1 +def test_info_without_loading_parquet_groups(tmp_path): + path = tmp_path / "info.parquet" + pq.write_table(pa.table({"id": [1, 2, 3], "text": ["a", "bb", "ccc"]}), path) + with blosc2.open(path, cache_dir=tmp_path / "cache") as remote: + before = remote.traffic.requests + info = dict(remote.info_items) + assert info["nbytes"] == "27 (27 B)" # Three int64 values plus the virtual validity mask. + assert info["cbytes"] == info["cratio"] == "n/a" + assert "id" in info["columns"] + assert remote.traffic.requests == before + + def test_unnamed_root_without_flattening(tmp_path): path = tmp_path / "root.parquet" source = pa.Table.from_arrays( From c5bfa568417684dabaa68f5033f264a3c9174625 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 13:47:52 +0200 Subject: [PATCH 05/19] Unify Parquet caching and speed warm reads --- doc/guides/remote_tables.md | 21 +- plans/remote-parquet-cache-metadata.md | 188 ++++++ plans/remote-parquet-cache-unification.md | 20 +- src/blosc2/remote_parquet.py | 748 ++++++++++++++++++---- src/blosc2/schunk.py | 5 + tests/ctable/test_remote_parquet.py | 194 +++++- 6 files changed, 1038 insertions(+), 138 deletions(-) create mode 100644 plans/remote-parquet-cache-metadata.md diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index 74d1db3a1..c8e473d86 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -35,15 +35,32 @@ null policies may need a first-batch sample. Flattening a single unnamed logical row count. `max_rows` stops that preparation at the requested prefix. `cache_dir` retains converted groups and the row map for later opens; without it, an in-memory LRU -is used. `refresh()` reopens the source and invalidates old views. `lazy=False` +is used. `refresh()` reopens a changed source and invalidates old views; it +returns without rebuilding the table when a disk-cached source is unchanged. `lazy=False` calls the eager `CTable.from_parquet()` importer. +The disk cache uses a readable source directory such as +`parquet-cache/readings.parquet--/.b2d/`. Each accessed +physical field and row group becomes a native CTable directory inside that +generation. Opening the generation path with `blosc2.open()` returns the whole +logical remote table and fetches missing groups from the source. +The generation also retains Parquet discovery metadata. A warm open restores +the schema and row-group map locally, without contacting the source. A small +local marker index locates the active source revision. Like other remote table +caches, it assumes that revision is immutable until `refresh()` is called. +The Arrow reader is created only when an uncached group is requested. +Cached row groups open directly from their native `.b2d` directories; warm +reads do not copy complete groups into memory. Reads against the shared seekable Parquet handle are serialized; increasing `max_concurrency` does not make one file's row-group reads parallel. HTTP servers must honor byte-range requests; fsspec raises a range-request error for servers that only return complete files. Download the file and import it locally when range access is unavailable. -Parquet disk caches require source size and an ETag or modification time. +Parquet disk caches require source size and a version marker. HTTP sources +can use an ETag, modification time, or Backblaze B2 file ID. +`table.save("reference.b2z")` creates a portable archive with the retained +groups; `blosc2.open("reference.b2z")` reuses those groups and fetches uncached +ones on demand. The earlier `reference.b2nd` format remains readable. `table.save("reference.b2nd")` includes already converted groups and the row map; use `include_cache=False` for a small cold reference. `blosc2.open()` reopens a reference when the source marker still matches. Runtime credentials diff --git a/plans/remote-parquet-cache-metadata.md b/plans/remote-parquet-cache-metadata.md new file mode 100644 index 000000000..abdba0227 --- /dev/null +++ b/plans/remote-parquet-cache-metadata.md @@ -0,0 +1,188 @@ +# Retain Parquet metadata for fast warm opens + +Status: implemented; ready for manual checks. Cold opens publish discovery +metadata in the generation manifest. A small local marker index finds that +generation on later opens without contacting the source. Warm opens restore +the schema before creating an Arrow reader. As with other remote table caches, +the source is assumed immutable until explicit `refresh()`. Existing caches +without the local marker index perform one source check before using this path. + +Observed in one Python process against the reported Backblaze file: cold +`open()` 1.53 s, warm `open()` 0.19 s. A reopened cached group took 0.23 s +including `open()` and transferred zero source bytes. These are single-run +measurements, not timing thresholds. + +Fresh processes using one temporary cache after the immutable-source +correction: cold `open()` 1.36 s with one HEAD and one GET (65,536 bytes read); +warm `open()` 0.029 s with no HTTP requests. A cold group read, including +`open()`, took 2.37 s with one GET; the reopened cached group took 0.051 s +with no HTTP requests. These timings exclude Python import time. + +## Objective + +Make a warm `blosc2.open(url, cache_dir=...)` reconstruct a Parquet table from +local discovery metadata. Opening the table and displaying `t.info` must avoid +downloading the footer, reading data samples, repeating schema inference, and +contacting the source. Check its version on cold open and explicit refresh. + +Keep the existing readable source directory, generation `.b2d` container, +ownership locks, converted-group cache, and portable `.b2z` exports. Store the +new discovery metadata in the generation's `embed.b2e` and a small marker +index at the cache root; no `parquet-sources/` directory is needed. + +## Existing code to reuse + +- `src/blosc2/remote_store.py`: `RemoteDiscovery._restore_manifest()` restores + discovery state before opening leaves; HDF5 reuses its parsed index. +- `src/blosc2/b2z_source.py`: `B2ZArchive` restores object information and + metadata ranges, avoiding repeated archive discovery. +- `src/blosc2/remote_store_cache.py`: `StoreDiskCache`, `SharedStoreCache`, + manifest publication, and generation ownership. +- `src/blosc2/schema_compiler.py`: `schema_to_dict()` and `schema_from_dict()` + preserve compiled schemas without repeating inference. +- `src/blosc2/remote_parquet.py`: retain its Arrow row-group adapter, source + marker validation, conversion semantics, and existing group-cache behavior. + +## Retained metadata + +Add a versioned Parquet discovery section to the existing manifest metadata. +It must contain enough information to construct the logical table without +creating a remote Arrow reader: + +- Source size and revision marker, including the Backblaze B2 file ID where + applicable. +- Arrow file metadata serialized using `FileMetaData.write_metadata_file()` + and restored using Arrow's metadata reader; do not use pickle. +- Compiled Blosc2 schema, physical/logical field mapping, and effective + conversion settings. +- Physical row-group boundaries and logical row boundaries/count. Preserve + the existing flattened-root row map when logical and physical rows differ. +- Cache/discovery compatibility version and the options identity used to + validate this state. + +Keep discovery metadata outside the converted-data eviction budget, consistent +with the other remote formats. Preserve meaningful metadata-size reporting. +Do not persist runtime credentials in the manifest. + +## Implementation sequence + +### 1. Resolve the cache before discovering the schema + +Refactor opening into local cache lookup, then either metadata restoration or +cold source validation and discovery. Cache lookup must not depend on +schema-derived values such as the inferred root name. + +Normalize the source and request options before lookup. Include projection, +row limit, reader options, effective null policy, and conversion/compression +options wherever they affect the restored table or retained groups. Preserve +the current separation of source revisions; moving revisions into generations +under a stable source directory is a separate lifecycle change. + +Define the identity compatibility path explicitly. Existing manifests lacking +discovery metadata should acquire it on the next discovery pass when their +identity remains compatible. The local marker index stores the last validated +revision and corresponding cache directory. If correcting the identity requires +a new cache directory, leave the old directory intact and document the one-time +rebuild. + +### 2. Consolidate source metadata requests + +For HTTP cold opens and explicit refresh, obtain size and supported version +headers in one metadata request using the existing fsspec session and transport +options. Capture ETag, +modification time, and Backblaze file ID without the current additional B2 HEAD. +For other filesystems, retain the appropriate `fs.info()` metadata path. + +Carry the validated size into later handle creation so HTTP fsspec does not +issue another HEAD solely to determine file size. Retain appropriate fallback +behavior for servers that cannot answer HEAD; the one-request target applies +to servers such as the reported Backblaze endpoint. + +### 3. Publish cold discovery and restore warm tables + +On a cold open, perform the existing footer read and necessary sample/row-map +discovery. Serialize the resulting state into the common manifest under its +existing lock and atomic publication mechanism. + +On a warm open, validate the discovery version and source/options identity, +restore the compiled schema and mappings, and construct `ParquetTableStorage` +directly. Avoid `CTable.from_arrow()` and remote data sampling on this path, +including for policies whose initial inference requires values. + +Validate stored schema, field mappings, counts, and row boundaries before +accepting them. Missing, incompatible, or malformed disposable discovery state +must trigger safe reconstruction; do not silently retain groups whose source +or conversion identity is uncertain. Avoid republishing unchanged manifests. + +### 4. Create the remote Arrow reader lazily + +Allow `_ParquetOwner` to hold restored metadata without an open remote handle +or `ParquetFile`. Information properties and cached-group reads must use that +state directly. + +Under the existing owner lock, create the handle and reader on the first +uncached-group request. Supply the retained `FileMetaData` through +`ParquetFile(metadata=...)` and preserve all applicable reader options. Verify +that this bypasses footer reads in the installed Arrow version. + +Make close and failed initialization safe both before and after reader +creation. Keep the existing serialization of reads through the shared handle. + +### 5. Integrate refresh, sharing, and saved artifacts + +Use the same restoration path for URL opens, generation-root opens, and saved +references. Include discovery metadata in `.save("reference.b2z")` even when +`include_cache=False`; export must not fetch unvisited groups. + +Preserve ordinary lifetime ownership and shared operation locks. Concurrent +opens must observe only fully published discovery metadata. A changed source +must not reuse old metadata or converted groups. Explicit refresh must perform +source validation and rebuild discovery when needed, preserve stale-view +semantics, and leave the old table usable if replacement preparation fails. + +Keep existing saved-reference compatibility. Older references without the new +section may perform discovery before benefiting from metadata retention. + +### 6. Document and measure the behavior + +Update `doc/guides/remote_tables.md` to explain retained discovery metadata, +the immutable-source assumption, lazy reader creation, compatibility, +and the distinction between discovery metadata and cached data groups. + +Measure the supplied Backblaze URL in fresh processes, separating Python +startup from `open()` time where useful: + +```text +https://f001.backblazeb2.com/file/blosc2/yellow_tripdata_2024-01.parquet +``` + +Report cold open, warm open with `t.info`, a cold group read, and a reopened +cached-group read. Count actual HTTP HEAD/GET requests as well as transferred +bytes: the existing table traffic counter does not include every metadata +request. Use request-count assertions for regression tests rather than brittle +wall-clock thresholds. + +## Acceptance checks + +- Warm URL open plus `t.info`: no HTTP request, no schema inference or sampling. +- Cold open: correct discovery and atomic retention, preserving existing + importer parity for nullable, nested, dictionary, and variable-length data. +- Cached-group read after reopen: no additional payload request or conversion. +- Uncached-group read: lazy reader creation uses retained metadata and fetches + only the data needed by the existing field/row-group access path. +- Projection, row limits, reader options, and conversion policies cannot reuse + incompatible restored table metadata. +- Flattened-root tables restore their logical row counts and row maps without + repeating the preparation scan. +- After explicit refresh, a source replacement, including one with the same + size but a different version marker, cannot reuse stale discovery or groups. +- Missing or corrupt discovery metadata, failed publication, and failed + refresh recover without exposing partially initialized table state. +- Shared opens, generation-root reopen, and exported references restore the + same schema and reuse retained groups. Cold exports still retain metadata. +- Closing a table before any data miss releases its cache ownership cleanly. + +Run focused Parquet regression tests and Ruff checks in the `blosc2` conda +environment. Run affected shared-cache/export tests if common code changes. +Finish with fresh-process measurements against the reported remote file and +report the observed warm-open improvement for manual checks. diff --git a/plans/remote-parquet-cache-unification.md b/plans/remote-parquet-cache-unification.md index 1b5cdad66..55699fe2b 100644 --- a/plans/remote-parquet-cache-unification.md +++ b/plans/remote-parquet-cache-unification.md @@ -1,6 +1,24 @@ # Unify Parquet with the remote table cache lifecycle -Proposal; no runtime changes implemented by this document. +Status: cache behavior ready for manual checks. The live cache uses readable +source directories, `StoreDiskCache` generations, native group `.b2d` payloads, +and `CacheCoordinator` accounting. Generation roots reopen as Parquet tables, +and `.save("reference.b2z")` exports retained groups as a single archive. + +Implementation decision: keep the Parquet owner and storage adapter for Arrow +row groups. `RemoteDiscovery` assumes existing Blosc2 array leaves for its +format dispatch, while Parquet creates converted table segments on demand. +Forcing that class to own Parquet would add format branches across discovery, +table storage, and export. The shared cache primitives now provide the intended +layout and lifecycle behavior. The deeper `RemoteDiscovery` integration below +remains an optional architectural follow-up rather than a requirement for +manual cache checks. + +The source revision is still part of Parquet's cache identity, so `refresh()` +uses a new readable source directory after a file changes. Moving that revision +into a new generation under a stable source directory is a separate lifecycle +change; the current behavior keeps old revisions isolated and old references +detect source changes. ## Objective diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index 0c50f0b62..decfb363b 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -3,14 +3,17 @@ from __future__ import annotations import hashlib +import json import os +import shutil import tempfile import threading import uuid import zipfile from collections import OrderedDict -from contextlib import contextmanager -from dataclasses import fields +from contextlib import nullcontext +from dataclasses import asdict, fields +from enum import Enum from pathlib import Path from urllib.parse import urlsplit, urlunsplit @@ -19,14 +22,18 @@ import blosc2 from blosc2.ctable import CTable, NullPolicy, get_null_policy, null_policy from blosc2.ctable_storage import RemoteTableStorage, _AllValidRows, split_field_path +from blosc2.proxy import CacheCoordinator from blosc2.proxy_source import Traffic from blosc2.remote_array import CACHE_POLICY_DEFAULT, normalize_cache_limit from blosc2.remote_ctable import RemoteCTable, _positive_integer -from blosc2.remote_store_cache import lock_cache_file -from blosc2.schema_compiler import schema_to_dict +from blosc2.schema_compiler import schema_from_dict, schema_to_dict def _portable(value): + if isinstance(value, blosc2.CParams | blosc2.DParams | NullPolicy): + value = asdict(value) + if isinstance(value, Enum): + value = value.value if isinstance(value, np.generic): value = value.item() if value is None or isinstance(value, bool | int | float | str | bytes): @@ -38,6 +45,84 @@ def _portable(value): raise TypeError(f"Nonportable Parquet reader or conversion option: {type(value).__name__}") +def _reference_options(owner): + options = { + key: value + for key, value in owner.reopen_kwargs.items() + if key + not in { + "storage_options", + "cache_dir", + "cache_policy", + "max_cache_bytes", + "shared_cache", + "_effective_null_policy", + "max_concurrency", + "metadata_buffer_bytes", + "row_buffer_bytes", + } + } + options["null_policy"] = { + field.name: getattr(owner.options["null_policy"], field.name) for field in fields(NullPolicy) + } + return _portable(options) + + +def _restore_compression_options(options): + for name, cls in (("cparams", blosc2.CParams), ("dparams", blosc2.DParams)): + if isinstance(options.get(name), dict): + options[name] = cls(**options[name]) + + +def _parquet_reopen_kwargs(values): + result = { + key: values[key] + for key in ( + "storage_options", + "columns", + "max_rows", + "parquet_options", + "string_max_length", + "auto_null_sentinels", + "null_storage", + "separate_nested_cols", + "list_serializer", + "blosc2_batch_size", + "blosc2_items_per_block", + "batch_size", + "cparams", + "dparams", + "validate", + "cache_policy", + "shared_cache", + "cache_dir", + ) + } + result["max_cache_bytes"] = values["max_cache_bytes"] + result["_effective_null_policy"] = values["effective_policy"] + result.update(values["settings"]) + return result + + +def _parquet_discovery(owner, schema, physical, length): + import pyarrow as pa + + sink = pa.BufferOutputStream() + owner.arrow_metadata.write_metadata_file(sink) + return { + "version": 1, + "arrow_metadata": sink.getvalue().to_pybytes(), + "schema": schema_to_dict(schema), + "physical": physical, + "row_ends": owner.row_ends.tolist(), + "length": length, + "options": { + **_portable({key: value for key, value in owner.options.items() if key != "null_policy"}), + "null_policy": asdict(owner.options["null_policy"]), + }, + } + + class _CountingHandle: def __init__(self, handle, traffic): self.handle = handle @@ -59,9 +144,18 @@ def __getattr__(self, name): def _source_marker(urlpath, storage_options): import fsspec + from fsspec.asyn import sync fs, path = fsspec.core.url_to_fs(urlpath, **(storage_options or {})) - info = fs.info(path) + if urlsplit(urlpath).scheme in {"http", "https"}: + try: + info = sync(fs.loop, _http_source_info, fs, path) + except (OSError, RuntimeError): + info = {} + if info.get("size") is None: + info = {**info, **fs.info(path)} + else: + info = fs.info(path) return { key: str(info[key]) for key in ( @@ -73,36 +167,140 @@ def _source_marker(urlpath, storage_options): "created", "LastModified", "Last-Modified", + "x-bz-file-id", ) if info.get(key) is not None } -def _disk_cache_path(urlpath, storage_options, options, cache_dir, marker): +async def _http_source_info(fs, path): + kwargs = fs.kwargs.copy() + allow_redirects = kwargs.pop("allow_redirects", True) + headers = kwargs.pop("headers", {}).copy() + headers["Accept-Encoding"] = "identity" + session = await fs.set_session() + async with session.head( + fs.encode_url(path), headers=headers, allow_redirects=allow_redirects, **kwargs + ) as response: + fs._raise_not_found_for_status(response, path) + info = {} + for key in ("Content-Length", "ETag", "Last-Modified", "x-bz-file-id"): + if response.headers.get(key): + if key == "Content-Length" and response.headers.get("Content-Encoding", "identity") not in { + "", + "identity", + }: + continue + info["size" if key == "Content-Length" else key] = response.headers[key] + return info + + +def _open_source_handle(urlpath, storage_options, marker, traffic): + import fsspec + + fs, path = fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + kwargs = ( + {"size": int(marker["size"])} if marker and urlsplit(urlpath).scheme in {"http", "https"} else {} + ) + return _CountingHandle(fs.open(path, "rb", **kwargs), traffic) + + +def _cache_marker_index(urlpath, storage_options, options, cache_dir): import pyarrow - if "size" not in marker or len(marker) < 2: - raise ValueError("A persistent Parquet cache requires source size and an ETag or modification time") - identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, marker, options) + if not urlsplit(urlpath).scheme: + urlpath = os.path.abspath(urlpath) + identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, options) digest = hashlib.sha256(repr(identity).encode()).hexdigest() - directory = Path(cache_dir) / "parquet" / digest - directory.mkdir(parents=True, exist_ok=True) - return directory + return Path(cache_dir) / f".parquet-marker-{digest}.json" + + +def _cached_source_marker(urlpath, storage_options, options, cache_dir): + path = _cache_marker_index(urlpath, storage_options, options, cache_dir) + try: + value = json.loads(path.read_text()) + marker, directory = value["marker"], value["directory"] + if ( + value["version"] == 1 + and isinstance(marker, dict) + and "size" in marker + and len(marker) > 1 + and isinstance(directory, str) + and directory == Path(directory).name + and (path.parent / directory / "active_generation.json").is_file() + ): + return marker + except (OSError, ValueError, KeyError, TypeError): + pass + return None + + +def _publish_source_marker(urlpath, storage_options, options, cache_dir, marker, disk): + from blosc2.remote_store_cache import atomic_write + + path = _cache_marker_index(urlpath, storage_options, options, cache_dir) + atomic_write(path, json.dumps({"version": 1, "marker": marker, "directory": disk.path.name}).encode()) -@contextmanager -def _disk_guard(directory): - if directory is None: - yield - return - with (directory / "owner.lock").open("a+b") as lock: - lock_cache_file(lock, blocking=True) - yield +def _disk_cache_path(urlpath, storage_options, options, cache_dir, marker, *, shared=False): + import pyarrow + + from blosc2.remote_store_cache import SharedStoreCache, StoreDiskCache + + if "size" not in marker or len(marker) < 2: + raise ValueError("A persistent Parquet cache requires source size and a version marker") + if not urlsplit(urlpath).scheme: + urlpath = os.path.abspath(urlpath) + identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, marker, options) + digest = hashlib.sha256(repr(identity).encode()).hexdigest() + url = urlsplit(urlpath) + source = { + "kind": "parquet", + "urlpath": urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")), + "identity": digest, + } + disk = (SharedStoreCache if shared else StoreDiskCache)(cache_dir, source) + try: + with _disk_guard(disk): + manifest = disk.load() + if manifest is None: + manifest = { + "version": 1, + "source": source, + "generation": uuid.uuid4().hex, + "nodes": {"": ("ctable", {})}, + "attrs": {}, + "listed": {}, + "metadata": {"source_marker": marker}, + "caches": [], + } + disk.publish(manifest) + return disk, disk.path / f"{manifest['generation']}.b2d" + except BaseException: + disk.close() + raise + + +def _disk_guard(disk): + return disk.guard() if disk is not None and hasattr(disk, "guard") else nullcontext() def _group_filename(number, physical): token = hashlib.sha256(physical.encode()).hexdigest()[:16] - return f"{number}-{token}.b2z" + return f"{number}-{token}.b2d" + + +def open_parquet_cache_artifact(path, *, mode="r", **kwargs): + if os.path.isdir(path): + carrier = blosc2.blosc2_ext.open(str(Path(path) / "embed.b2e"), "r", 0) + else: + with zipfile.ZipFile(path) as archive: + carrier = blosc2.schunk_from_cframe(archive.read("embed.b2e")) + if carrier.vlmeta["b2remote_manifest"]["source"].get("kind") != "parquet": + return None + if os.path.isdir(path): + return RemoteParquetCTable._open_cache_artifact(path, mode=mode, **kwargs) + return RemoteParquetCTable._open_archive_artifact(path, mode=mode, **kwargs) def _projected_field(name, physical, paths): @@ -124,10 +322,29 @@ class _ParquetOwner: is_mutable = True shared = False - def __init__(self, path, handle, parquet_file, options, *, cache_policy, max_cache_bytes, cache_dir): + def __init__( + self, + path, + handle, + parquet_file, + options, + *, + cache_policy, + max_cache_bytes, + cache_dir, + disk, + storage_options=None, + source_marker=None, + arrow_metadata=None, + parquet_options=None, + ): self.urlpath = path self.handle = handle - self.traffic = handle.traffic + self.traffic = handle.traffic if handle is not None else Traffic() + self.storage_options = storage_options + self.source_marker = source_marker + self.arrow_metadata = arrow_metadata or parquet_file.metadata + self.parquet_options = parquet_options self.parquet_file = parquet_file self.options = options self.lock = threading.RLock() @@ -139,9 +356,17 @@ def __init__(self, path, handle, parquet_file, options, *, cache_policy, max_cac self.cache_policy = cache_policy self.max_cache_bytes = max_cache_bytes self.cache_dir = cache_dir + self.disk = disk + self._cache_key = "parquet" + self._cache_sizes = {} + self._cache_lru = OrderedDict() + self.cache_coordinator = CacheCoordinator(max_cache_bytes) if disk is not None else None + if self.cache_coordinator is not None: + self._sync_evictions() + self.cache_coordinator.register(self) self._cache_manager = None self.row_ends = np.cumsum( - [parquet_file.metadata.row_group(i).num_rows for i in range(parquet_file.num_row_groups)] + [self.arrow_metadata.row_group(i).num_rows for i in range(self.arrow_metadata.num_row_groups)] ) def acquire(self): @@ -152,17 +377,68 @@ def release(self): if self.users == 0: self.closed = True self.cache.clear() - self.parquet_file.close() - self.handle.close() + if self.parquet_file is not None: + self.parquet_file.close() + if self.handle is not None: + self.handle.close() + if self.disk is not None: + self.disk.close() if self._cache_manager is not None: self._cache_manager.cleanup() def group(self, number, physical): key = number, physical with self.lock: - with _disk_guard(self.cache_dir): + with _disk_guard(self.disk): return self._group_locked(number, physical, key) + def _sync_evictions(self): + if self.cache_dir is None: + return + # ponytail: scan cached groups on each accounting pass; persist sizes if directory walks dominate reads. + entries = sorted(self.cache_dir.glob("*-*.b2d"), key=lambda path: path.stat().st_mtime_ns) + present = {path.name for path in entries} + for name in tuple(self._cache_sizes): + if name not in present: + self._cache_sizes.pop(name) + self._cache_lru.pop(name, None) + if self.cache_coordinator is not None: + self.cache_coordinator.forget(self, name) + for path in entries: + if path.name not in self._cache_sizes: + self._cache_sizes[path.name] = sum( + file.stat().st_size for file in path.rglob("*") if file.is_file() + ) + self._cache_lru[path.name] = None + if self.cache_coordinator is not None: + self.cache_coordinator.touch(self, path.name) + + def _retained_cache_bytes(self): + return sum(self._cache_sizes.values()) + + def _trim_cache(self, target_bytes, *, max_chunks=None): + evicted = [] + while ( + self._retained_cache_bytes() > target_bytes + and self._cache_lru + and (max_chunks is None or len(evicted) < max_chunks) + ): + name = next(iter(self._cache_lru)) + shutil.rmtree(self.cache_dir / name) + self._cache_lru.pop(name) + self._cache_sizes.pop(name) + self.cache_coordinator.forget(self, name) + evicted.append(name) + return tuple(evicted) + + def _touch_group(self, path): + if self.cache_coordinator is None: + return + path.touch() + self._cache_lru.pop(path.name, None) + self._cache_lru[path.name] = None + self.cache_coordinator.touch(self, path.name) + def _group_locked(self, number, physical, key): if key in self.cache: self.cache.move_to_end(key) @@ -172,13 +448,28 @@ def _group_locked(self, number, physical, key): cache_file = self.cache_dir / _group_filename(number, physical) if cache_file.exists(): try: - cached = CTable.load(cache_file) + cached = CTable.open(cache_file) start = 0 if number == 0 else int(self.row_ends[number - 1]) if len(cached) == int(self.row_ends[number]) - start: + self._touch_group(cache_file) return cached cached.close() except (OSError, ValueError, RuntimeError, KeyError, TypeError, zipfile.BadZipFile): pass # Interrupted or corrupt publication: rebuild this group. + if self.parquet_file is None: + self.handle = _open_source_handle( + self.urlpath, self.storage_options, self.source_marker, self.traffic + ) + try: + import pyarrow.parquet as pq + + self.parquet_file = pq.ParquetFile( + self.handle, metadata=self.arrow_metadata, **(self.parquet_options or {}) + ) + except BaseException: + self.handle.close() + self.handle = None + raise arrow = self.parquet_file.read_row_group(number, columns=[physical], use_threads=False) if physical == "" and not self.options["flatten_root"]: arrow = arrow.rename_columns([self.options["root_name"]]) @@ -203,14 +494,19 @@ def _group_locked(self, number, physical, key): ) size = int(getattr(table, "cbytes", 0) or arrow.nbytes) if cache_file is not None: - temporary = cache_file.with_name(f".{uuid.uuid4().hex}.b2z") + temporary = cache_file.with_name(f".{uuid.uuid4().hex}.b2d") try: with table.copy(urlpath=temporary): pass + if cache_file.exists(): + shutil.rmtree(cache_file) os.replace(temporary, cache_file) finally: - temporary.unlink(missing_ok=True) - self._evict_disk(cache_file) + if temporary.exists(): + shutil.rmtree(temporary) + self._sync_evictions() + self._touch_group(cache_file) + self.cache_coordinator.enforce() elif self.cache_policy is blosc2.CachePolicy.MEMORY: self.cache[key] = table, size self.cache_bytes += size @@ -223,20 +519,6 @@ def _group_locked(self, number, physical, key): self.cache_bytes -= removed return table - def _evict_disk(self, newest): - if self.max_cache_bytes is None: - return - entries = sorted(self.cache_dir.glob("*.b2z"), key=lambda path: path.stat().st_mtime_ns) - total = sum(path.stat().st_size for path in entries) - total += sum(path.stat().st_size for path in self.cache_dir.glob("row-map-*.npy")) - for path in entries: - if total <= self.max_cache_bytes: - break - if path != newest: - size = path.stat().st_size - path.unlink(missing_ok=True) - total -= size - class _ParquetColumn: _compressed_size_unavailable = True @@ -429,13 +711,13 @@ def __new__( # noqa: C901 _effective_null_policy=None, **kwargs, ): - import fsspec import pyarrow as pa import pyarrow.parquet as pq cache_dir = kwargs.pop("cache_dir", None) seed_cache = kwargs.pop("_seed_cache", None) cache_manager = kwargs.pop("_cache_manager", None) + refresh_marker = kwargs.pop("_refresh_marker", None) if kwargs: raise TypeError(f"Unsupported Parquet options: {', '.join(kwargs)}") if max_rows is not None and max_rows < 0: @@ -462,8 +744,99 @@ def __new__( # noqa: C901 if shared_cache and cache_policy is not blosc2.CachePolicy.DISK: raise ValueError("shared_cache=True requires a disk cache") max_cache_bytes = normalize_cache_limit(cache_policy, max_cache_bytes) - handle = _CountingHandle(fsspec.open(urlpath, "rb", **(storage_options or {})).open(), Traffic()) + effective_policy = _effective_null_policy or get_null_policy() + options = { + "null_policy": effective_policy, + "string_max_length": string_max_length, + "auto_null_sentinels": auto_null_sentinels, + "null_storage": null_storage, + "separate_nested_cols": separate_nested_cols, + "list_serializer": list_serializer, + "blosc2_batch_size": blosc2_batch_size, + "blosc2_items_per_block": blosc2_items_per_block, + "batch_size": batch_size, + "cparams": cparams, + "dparams": dparams, + "validate": validate, + } + identity_options = (columns, max_rows, parquet_options, _portable(options)) + cached_marker = ( + _cached_source_marker(urlpath, storage_options, identity_options, cache_dir) + if cache_dir is not None and refresh_marker is None + else None + ) + marker = ( + refresh_marker or cached_marker or _source_marker(urlpath, storage_options) + if cache_dir is not None + else None + ) + disk, disk_dir = ( + _disk_cache_path( + urlpath, storage_options, identity_options, cache_dir, marker, shared=shared_cache + ) + if cache_dir is not None + else (None, None) + ) + handle = None try: + discovery = disk.load()["metadata"].get("parquet_discovery") if disk is not None else None + if discovery is not None: + try: + schema = schema_from_dict(discovery["schema"]) + physical = discovery["physical"] + ends = np.asarray(discovery["row_ends"], dtype=np.int64) + length = discovery["length"] + metadata = pq.read_metadata(pa.BufferReader(discovery["arrow_metadata"])) + if ( + discovery["version"] != 1 + or not isinstance(physical, dict) + or set(physical) != {column.name for column in schema.columns} + or not isinstance(length, int) + or length < 0 + or ends.ndim != 1 + or len(ends) > metadata.num_row_groups + or np.any(ends < 0) + or np.any(ends[1:] < ends[:-1]) + or (len(ends) and length > ends[-1]) + ): + raise ValueError("Invalid retained Parquet metadata") + restored_options = dict(discovery["options"]) + restored_options["null_policy"] = NullPolicy(**restored_options["null_policy"]) + _restore_compression_options(restored_options) + except (KeyError, TypeError, ValueError, OSError): + discovery = None + else: + owner = _ParquetOwner( + urlpath, + None, + None, + restored_options, + cache_policy=cache_policy, + max_cache_bytes=max_cache_bytes, + cache_dir=disk_dir, + disk=disk, + storage_options=storage_options, + source_marker=marker, + arrow_metadata=metadata, + parquet_options=parquet_options, + ) + owner.row_ends = ends + owner.identity_options = identity_options + owner.discovery = discovery + owner._cache_manager = cache_manager + owner.reopen_kwargs = _parquet_reopen_kwargs(locals()) + storage = ParquetTableStorage(owner, schema, physical, length, **settings) + try: + table = cls._open_from_storage(storage) + if cached_marker is None: + _publish_source_marker( + urlpath, storage_options, identity_options, cache_dir, marker, disk + ) + return table + except BaseException: + storage.close() + raise + handle = _open_source_handle(urlpath, storage_options, marker, Traffic()) pf = pq.ParquetFile(handle, **(parquet_options or {})) fields = pf.schema_arrow if columns is not None: @@ -473,7 +846,6 @@ def __new__( # noqa: C901 while root_name in fields.names: root_name += "_1" flatten_root = separate_nested_cols and CTable._detect_unnamed_root_list_struct(pa, fields) - effective_policy = _effective_null_policy or get_null_policy() # Mask-backed schemas are determined by the Arrow schema alone. A # value sample is only needed when in-band null handling may reject # actual nulls during inference. @@ -493,35 +865,12 @@ def __new__( # noqa: C901 meta = dict(sample.schema.metadata or {}) meta[b"blosc2_empty_root_physical"] = root_name.encode() sample = sample.replace_schema_metadata(meta) - options = { - "root_name": root_name, - "flatten_root": flatten_root, - "null_policy": effective_policy, - "string_max_length": string_max_length, - "auto_null_sentinels": auto_null_sentinels, - "null_storage": null_storage, - "separate_nested_cols": separate_nested_cols, - "list_serializer": list_serializer, - "blosc2_batch_size": blosc2_batch_size, - "blosc2_items_per_block": blosc2_items_per_block, - "batch_size": batch_size, - "cparams": cparams, - "dparams": dparams, - "validate": validate, - } - # A plain in-memory read needs only the file handle's metadata. - # Fetch a stronger source marker when disk caching or saving a - # portable reference actually needs one. - marker = _source_marker(urlpath, storage_options) if cache_dir is not None else None - disk_dir = ( - _disk_cache_path(urlpath, storage_options, options, cache_dir, marker) - if cache_dir is not None - else None - ) + options["root_name"] = root_name + options["flatten_root"] = flatten_root if seed_cache: if disk_dir is None: raise ValueError("A retained Parquet cache requires cache_dir") - with _disk_guard(disk_dir): + with _disk_guard(disk): for name, data in seed_cache.items(): if ( not isinstance(name, str) @@ -530,11 +879,18 @@ def __new__( # noqa: C901 or Path(name).name != name ): raise ValueError("Invalid retained Parquet cache entry") - destination = disk_dir / name - with tempfile.NamedTemporaryFile(dir=disk_dir, delete=False) as staged: + destination = disk_dir / (name[:-4] + ".b2d" if name.endswith(".b2z") else name) + with tempfile.NamedTemporaryFile( + dir=disk_dir, suffix=Path(name).suffix, delete=False + ) as staged: staged.write(data) try: - os.replace(staged.name, destination) + if name.endswith(".b2z"): + with CTable.load(staged.name) as table: + with table.copy(urlpath=destination): + pass + else: + os.replace(staged.name, destination) finally: Path(staged.name).unlink(missing_ok=True) with null_policy(options["null_policy"]): @@ -571,7 +927,7 @@ def __new__( # noqa: C901 } if flatten_root: # ponytail: full length scan; replace with a persisted offsets map when large nested roots matter. - with _disk_guard(disk_dir): + with _disk_guard(disk): lengths = [] count = 0 map_file = disk_dir / f"row-map-{max_rows}.npy" if disk_dir is not None else None @@ -629,35 +985,28 @@ def __new__( # noqa: C901 cache_policy=cache_policy, max_cache_bytes=max_cache_bytes, cache_dir=disk_dir, + disk=disk, + storage_options=storage_options, + source_marker=marker, + parquet_options=parquet_options, ) owner.source_marker = marker + owner.identity_options = identity_options owner._cache_manager = cache_manager - owner.reopen_kwargs = { - "storage_options": storage_options, - "columns": columns, - "max_rows": max_rows, - "parquet_options": parquet_options, - "string_max_length": string_max_length, - "auto_null_sentinels": auto_null_sentinels, - "null_storage": null_storage, - "separate_nested_cols": separate_nested_cols, - "list_serializer": list_serializer, - "blosc2_batch_size": blosc2_batch_size, - "blosc2_items_per_block": blosc2_items_per_block, - "batch_size": batch_size, - "cparams": cparams, - "dparams": dparams, - "validate": validate, - "max_cache_bytes": CACHE_POLICY_DEFAULT - if cache_policy is blosc2.CachePolicy.NONE - else max_cache_bytes, - "cache_policy": cache_policy, - "shared_cache": shared_cache, - "cache_dir": cache_dir, - "_effective_null_policy": effective_policy, - **settings, - } + owner.reopen_kwargs = _parquet_reopen_kwargs(locals()) owner.row_ends = ends + owner.discovery = _parquet_discovery(owner, probe._schema, physical, length) + if disk is not None: + with _disk_guard(disk): + manifest = disk.load() + try: + manifest["metadata"]["reopen"] = _reference_options(owner) + except TypeError: + manifest["metadata"]["reopen"] = None + manifest["metadata"]["parquet_discovery"] = owner.discovery + disk.publish(manifest) + if disk is not None and cached_marker is None: + _publish_source_marker(urlpath, storage_options, identity_options, cache_dir, marker, disk) storage = ParquetTableStorage(owner, probe._schema, physical, length, **settings) probe.close() try: @@ -666,14 +1015,22 @@ def __new__( # noqa: C901 storage.close() raise except BaseException: - handle.close() + if handle is not None: + handle.close() + if disk is not None: + disk.close() raise def refresh(self): storage = self._remote_storage() owner = storage._owner with owner.lock: - fresh = type(self)(owner.urlpath, **owner.reopen_kwargs) + refresh_marker = ( + _source_marker(owner.urlpath, owner.storage_options) if owner.disk is not None else None + ) + if refresh_marker is not None and refresh_marker == owner.source_marker: + return + fresh = type(self)(owner.urlpath, _refresh_marker=refresh_marker, **owner.reopen_kwargs) fresh._remote_storage()._owner._cache_manager = owner._cache_manager owner._cache_manager = None owner.generation += 1 @@ -701,13 +1058,17 @@ def cache_bytes(self): owner = self._remote_storage()._owner if owner.cache_dir is not None: return sum( - path.stat().st_size for path in owner.cache_dir.iterdir() if path.suffix in {".b2z", ".npy"} + file.stat().st_size + for path in owner.cache_dir.iterdir() + if path.suffix in {".b2d", ".npy"} + for file in (path.rglob("*") if path.is_dir() else (path,)) + if file.is_file() ) return owner.cache_bytes @property def metadata_bytes(self): - return self._remote_storage()._owner.parquet_file.metadata.serialized_size + return self._remote_storage()._owner.arrow_metadata.serialized_size @property def mutable(self): @@ -722,8 +1083,140 @@ def is_cache_mutable(self): return True def save(self, *args, **kwargs): + destination = args[0] if args else kwargs.get("destination") + if destination is not None and str(destination).endswith(".b2z"): + return self._save_archive(*args, **kwargs) return self._save_reference(*args, **kwargs) + def _save_archive(self, destination, *, include_cache=True, overwrite=False, mutable=None): + from blosc2.remote_array import validate_persistable_url + + if mutable not in (None, False): + raise ValueError("Parquet references are read-only") + owner = self._remote_storage()._owner + if urlsplit(owner.urlpath).scheme: + validate_persistable_url(owner.urlpath) + destination = Path(destination) + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if owner.disk is not None and destination.resolve().is_relative_to(owner.disk.path.resolve()): + raise ValueError("destination cannot be inside the live cache") + if owner.source_marker is None: + owner.source_marker = _source_marker(owner.urlpath, owner.reopen_kwargs["storage_options"]) + with owner.lock, _disk_guard(owner.disk), tempfile.TemporaryDirectory() as temporary: + if owner.disk is None: + disk, root = _disk_cache_path( + owner.urlpath, + owner.reopen_kwargs["storage_options"], + owner.identity_options, + temporary, + owner.source_marker, + ) + try: + manifest = disk.load() + manifest["metadata"]["reopen"] = _reference_options(owner) + manifest["metadata"]["parquet_discovery"] = owner.discovery + disk.publish(manifest) + if include_cache: + for (number, physical), (table, _) in owner.cache.items(): + with table.copy(urlpath=root / _group_filename(number, physical)): + pass + finally: + disk.close() + else: + root = owner.cache_dir + manifest = owner.disk.load() + if manifest["metadata"].get("reopen") is None: + raise TypeError("Parquet conversion options cannot be saved in a portable archive") + staged = Path(temporary) / "reference.b2z" + with zipfile.ZipFile(staged, "w", zipfile.ZIP_STORED) as archive: + archive.write(root.parent / "active_generation.json", "active_generation.json") + archive.write(root / "embed.b2e", "embed.b2e") + if include_cache: + for file in root.rglob("*"): + if file.is_file() and not file.name.startswith(".") and file != root / "embed.b2e": + archive.write(file, f"{root.name}/{file.relative_to(root)}") + destination.parent.mkdir(parents=True, exist_ok=True) + os.replace(staged, destination) + return str(destination) + + @classmethod + def _open_cache_artifact(cls, path, *, mode="r", storage_options=None, source_url=None): + if mode != "r": + raise ValueError("Parquet caches are read-only") + path = Path(path).resolve() + carrier = blosc2.blosc2_ext.open(str(path / "embed.b2e"), "r", 0) + manifest = carrier.vlmeta["b2remote_manifest"] + source = manifest["source"] + if source.get("kind") != "parquet": + raise ValueError("Cache is not a Parquet table") + if urlsplit(source["urlpath"]).scheme: + from blosc2.remote_array import validate_persistable_url + + validate_persistable_url(source["urlpath"]) + options = manifest["metadata"].get("reopen") + if options is None: + raise ValueError("This Parquet cache needs the original conversion options to reopen") + options = dict(options) + policy = options.pop("null_policy") + _restore_compression_options(options) + urlpath = source_url or source["urlpath"] + table = cls( + urlpath, + cache_dir=path.parent.parent, + storage_options=storage_options, + _effective_null_policy=NullPolicy(**policy), + _refresh_marker=manifest["metadata"]["source_marker"], + **options, + ) + if table._remote_storage()._owner.disk.path != path.parent: + table.close() + raise ValueError("Parquet cache source or options no longer match this generation") + return table + + @classmethod + def _open_archive_artifact(cls, path, *, mode="r", storage_options=None, source_url=None): + from blosc2.remote_store_cache import StoreDiskCache + + manager = tempfile.TemporaryDirectory(prefix="parquet-reference-") + try: + with zipfile.ZipFile(path) as archive: + carrier = blosc2.schunk_from_cframe(archive.read("embed.b2e")) + manifest = carrier.vlmeta["b2remote_manifest"] + source = manifest["source"] + if source.get("kind") != "parquet": + raise ValueError("Cache archive is not a Parquet table") + generation = f"{manifest['generation']}.b2d" + cache_root = StoreDiskCache.path_for(manager.name, source) + cache_root.mkdir(parents=True) + for member in archive.infolist(): + name = member.filename + if ( + ( + name not in {"active_generation.json", "embed.b2e"} + and not name.startswith(generation + "/") + ) + or ".." in Path(name).parts + or member.is_dir() + or member.compress_type != zipfile.ZIP_STORED + ): + raise ValueError("Invalid Parquet cache archive member") + target = cache_root / (f"{generation}/embed.b2e" if name == "embed.b2e" else name) + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(member) as source_file, target.open("wb") as target_file: + shutil.copyfileobj(source_file, target_file) + table = cls._open_cache_artifact( + cache_root / generation, + mode=mode, + storage_options=storage_options, + source_url=source_url, + ) + table._remote_storage()._owner._cache_manager = manager + return table + except BaseException: + manager.cleanup() + raise + def _save_reference(self, destination, *, include_cache=True, overwrite=False, mutable=None): """Write a CFrame-backed source reference without runtime credentials.""" from blosc2.b2objects import make_b2object_carrier, write_b2object_payload @@ -739,24 +1232,7 @@ def _save_reference(self, destination, *, include_cache=True, overwrite=False, m owner.source_marker = _source_marker(url, owner.reopen_kwargs["storage_options"]) if "size" not in owner.source_marker or len(owner.source_marker) < 2: raise ValueError("A portable Parquet reference requires an ETag or modification time") - options = { - key: value - for key, value in owner.reopen_kwargs.items() - if key - not in { - "storage_options", - "cache_dir", - "cache_policy", - "max_cache_bytes", - "shared_cache", - "_effective_null_policy", - } - and key not in {"max_concurrency", "metadata_buffer_bytes", "row_buffer_bytes"} - } - options["null_policy"] = { - field.name: getattr(owner.options["null_policy"], field.name) for field in fields(NullPolicy) - } - options = _portable(options) + options = _reference_options(owner) payload = { "kind": "remote_parquet", "version": 1, @@ -766,17 +1242,24 @@ def _save_reference(self, destination, *, include_cache=True, overwrite=False, m } if include_cache: retained = {} - with owner.lock, _disk_guard(owner.cache_dir): + with owner.lock, _disk_guard(owner.disk): if owner.cache_dir is not None: - retained.update( - (path.name, path.read_bytes()) - for path in owner.cache_dir.iterdir() - if path.suffix in {".b2z", ".npy"} - ) + with tempfile.TemporaryDirectory() as cache_export: + for path in owner.cache_dir.iterdir(): + if path.suffix == ".npy": + retained[path.name] = path.read_bytes() + elif path.suffix == ".b2d": + exported = Path(cache_export) / f"{path.stem}.b2z" + with CTable.open(str(path)) as table, table.copy(urlpath=exported): + pass + retained[exported.name] = exported.read_bytes() else: with tempfile.TemporaryDirectory() as cache_export: for (number, physical), (table, _) in owner.cache.items(): - path = Path(cache_export) / _group_filename(number, physical) + path = ( + Path(cache_export) + / f"{number}-{hashlib.sha256(physical.encode()).hexdigest()[:16]}.b2z" + ) with table.copy(urlpath=path): pass retained[path.name] = path.read_bytes() @@ -818,6 +1301,7 @@ def _from_payload(cls, payload, *, storage_options=None, parquet_options=None): policy = options.pop("null_policy", None) if not isinstance(policy, dict): raise ValueError("Parquet reference has no null policy") + _restore_compression_options(options) saved_reader = options.get("parquet_options") or {} if parquet_options: for key, value in parquet_options.items(): diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 768ba09b0..dc887c815 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1847,6 +1847,11 @@ def _open_special_store(urlpath, mode, offset, **kwargs): if "b2remote_store" in schunk_meta: if offset != 0: raise ValueError("Offset must be 0 for RemoteStore") + from blosc2.remote_parquet import open_parquet_cache_artifact + + parquet = open_parquet_cache_artifact(urlpath, mode=mode, **kwargs) + if parquet is not None: + return parquet from blosc2.remote_store import RemoteStore return RemoteStore._open_artifact(urlpath, mode=mode, **kwargs) diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index 376eabb6c..f17e2cc21 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -16,6 +16,8 @@ import pytest import blosc2 +from blosc2 import remote_parquet +from blosc2.ctable import CTable from blosc2.schema_compiler import schema_to_dict @@ -485,6 +487,125 @@ def test_disk_cache_reuses_group_and_root_map(tmp_path): assert reopened_requests < prepared_requests # row map reused assert second["id"][-1] == 5 assert second.traffic.requests == reopened_requests # converted group reused + cache_root = next(cache_dir.glob("root.parquet--*")) + generation = next(cache_root.glob("*.b2d")) + with blosc2.open(generation) as reopened: + assert reopened.col_names == ["id"] + before = reopened.traffic.requests + assert reopened["id"][-1] == 5 + assert reopened.traffic.requests == before + + +def test_warm_open_uses_retained_metadata_and_lazily_opens_arrow(tmp_path, monkeypatch): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir) as cold: + assert cold["x"][0] == 1 + + real_reader = pq.ParquetFile + readers = [] + + def record_reader(*args, **kwargs): + assert kwargs.get("metadata") is not None + readers.append(kwargs["metadata"]) + return real_reader(*args, **kwargs) + + monkeypatch.setattr(pq, "ParquetFile", record_reader) + real_from_arrow = CTable.from_arrow + monkeypatch.setattr(CTable, "from_arrow", lambda *args, **kwargs: pytest.fail("schema was inferred")) + monkeypatch.setattr(CTable, "load", lambda *args, **kwargs: pytest.fail("cached group was copied")) + monkeypatch.setattr(remote_parquet, "_source_marker", lambda *args: pytest.fail("source was checked")) + with blosc2.open(path, cache_dir=cache_dir) as warm: + assert warm.nrows == 4 + assert warm.metadata_bytes > 0 + assert readers == [] + assert warm["x"][0] == 1 + assert readers == [] + monkeypatch.setattr(CTable, "from_arrow", real_from_arrow) + assert warm["y"][3] == 8 + assert len(readers) == 1 + + +def test_retained_metadata_is_separate_for_projections(tmp_path): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2], "y": [3, 4]}), path) + with blosc2.open(path, cache_dir=cache_dir, columns=["x"]) as x_only: + assert x_only.col_names == ["x"] + with blosc2.open(path, cache_dir=cache_dir, columns=["y"]) as y_only: + assert y_only.col_names == ["y"] + assert len(list(cache_dir.glob("source.parquet--*"))) == 2 + + +def test_invalid_retained_metadata_is_rebuilt(tmp_path): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, cache_dir=cache_dir) as table: + disk = table._remote_storage()._owner.disk + manifest = disk.load() + manifest["metadata"]["parquet_discovery"]["schema"] = {"version": 999} + disk.publish(manifest) + with blosc2.open(path, cache_dir=cache_dir) as rebuilt: + assert rebuilt["x"][0] == 1 + assert rebuilt._remote_storage()._owner.disk.load()["metadata"]["parquet_discovery"]["version"] == 1 + + +def test_existing_cache_adds_local_marker_index(tmp_path, monkeypatch): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, cache_dir=cache_dir): + pass + index = next(cache_dir.glob(".parquet-marker-*.json")) + index.unlink() + with blosc2.open(path, cache_dir=cache_dir): + pass + assert index.is_file() + monkeypatch.setattr(remote_parquet, "_source_marker", lambda *args: pytest.fail("source was checked")) + with blosc2.open(path, cache_dir=cache_dir) as warm: + assert warm.nrows == 2 + + +@pytest.mark.parametrize("disk", [False, True]) +def test_cache_archive_reuses_warm_group_and_fetches_cold_group(tmp_path, disk): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) + options = {"cache_dir": tmp_path / "cache"} if disk else {} + artifact = tmp_path / "reference.b2z" + cold_artifact = tmp_path / "cold-reference.b2z" + with blosc2.open(path, **options) as original: + assert original["x"][0] == 1 + original.save(artifact) + original.save(cold_artifact, include_cache=False) + with blosc2.open(artifact) as restored: + assert restored.col_names == ["x", "y"] + before = restored.traffic.requests + assert restored["x"][0] == 1 + assert restored.traffic.requests == before + assert restored["y"][3] == 8 + assert restored.traffic.requests > before + with blosc2.open(cold_artifact) as cold: + before = cold.traffic.requests + assert cold["x"][0] == 1 + assert cold.traffic.requests > before + result = subprocess.check_output( + [sys.executable, "-c", "import blosc2,sys; print(blosc2.open(sys.argv[1])['x'][0])", str(artifact)], + text=True, + ) + assert result.strip() == "1" + + +def test_cache_generation_reopens_with_compression_options(tmp_path): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2, 3]}), path) + cache_dir = tmp_path / "cache" + with blosc2.open(path, cache_dir=cache_dir, cparams=blosc2.CParams(clevel=1)) as original: + assert original["x"][0] == 1 + generation = next(cache_dir.glob("*/*.b2d")) + with blosc2.open(generation) as reopened: + assert reopened["x"][0] == 1 def test_shared_disk_cache_reuses_converted_group_across_processes(tmp_path): @@ -525,9 +646,14 @@ def test_disk_cache_eviction_and_corrupt_entry(tmp_path): with blosc2.open(path, cache_dir=cache_dir, max_cache_bytes=1) as remote: assert remote["id"][2] == 2 assert remote["id"][5] == 5 - entries = list(cache_dir.rglob("*.b2z")) - assert len(entries) == 1 # one indivisible converted group exceeds the budget - entries[0].write_bytes(b"broken") + entries = list(cache_dir.rglob("*.b2d")) + entries = [entry for entry in entries if entry.name.startswith(("0-", "1-", "2-"))] + assert not entries # The common cache budget does not retain an oversized group. + with blosc2.open(path, cache_dir=cache_dir, max_cache_bytes=None) as remote: + assert remote["id"][5] == 5 + entries = [entry for entry in cache_dir.rglob("*.b2d") if entry.name.startswith("2-")] + assert len(entries) == 1 + (entries[0] / "_meta.b2f").write_bytes(b"broken") with blosc2.open(path, cache_dir=cache_dir) as remote: assert remote["id"][5] == 5 @@ -543,6 +669,68 @@ def test_refresh_disk_cache_uses_new_generation(tmp_path): assert remote["id"][0] == 3 +def test_same_size_source_replacement_rebuilds_metadata(tmp_path): + path = tmp_path / "changing.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": [1, 2]}), path) + old = path.stat() + with blosc2.open(path, cache_dir=cache_dir) as first: + assert first["id"][0] == 1 + pq.write_table(pa.table({"id": [3, 4]}), path) + assert path.stat().st_size == old.st_size + os.utime(path, ns=(old.st_atime_ns, old.st_mtime_ns + 2_000_000_000)) + with blosc2.open(path, cache_dir=cache_dir) as replaced: + assert replaced["id"][0] == 1 # Cached sources stay immutable until refresh. + replaced.refresh() + assert replaced["id"][0] == 3 + + +def test_refresh_unchanged_disk_source_keeps_cache_owner(tmp_path): + path = tmp_path / "unchanged.parquet" + pq.write_table(pa.table({"id": [1, 2]}), path) + with blosc2.open(path, cache_dir=tmp_path / "cache") as remote: + remote.refresh() + assert remote["id"][0] == 1 + + +def test_http_b2_file_id_supplies_missing_cache_validator(monkeypatch): + class Response: + def __init__(self): + self.headers = {"Content-Length": "49961641", "x-bz-file-id": "file-version-1"} + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + class Session: + def head(self, *args, **kwargs): + return Response() + + class Filesystem: + loop = fsspec.asyn.get_loop() + + def __init__(self): + self.kwargs = {} + + def info(self, path): + pytest.fail("HTTP source information required a second request") + + async def set_session(self): + return Session() + + def encode_url(self, path): + return path + + def _raise_not_found_for_status(self, response, path): + pass + + monkeypatch.setattr(fsspec.core, "url_to_fs", lambda *args, **kwargs: (Filesystem(), args[0])) + marker = remote_parquet._source_marker("https://example.com/table.parquet", None) + assert marker == {"size": "49961641", "x-bz-file-id": "file-version-1"} + + def test_http_range_requests_are_narrow(tmp_path): path = tmp_path / "served.parquet" pq.write_table( From a025412520f837d2d5b5f3c743701c52548e302d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 13:51:57 +0200 Subject: [PATCH 06/19] Simplify remote Parquet guide --- doc/guides/remote_tables.md | 59 ++++++++-------------------------- doc/reference/remotectable.rst | 21 ++++++++++-- 2 files changed, 33 insertions(+), 47 deletions(-) diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index c8e473d86..3e7c73294 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -25,55 +25,24 @@ with blosc2.open( local = table.copy(urlpath="readings.b2z") ``` -Reads convert the requested physical field and row group using the Arrow -importer. Nested struct leaves are projected from Parquet when their physical -paths are unambiguous. A narrow scalar read can avoid unrelated fields and groups; a group -with an indivisible large value can still allocate much more than the cache -budget. With mask-backed nulls, schema inference reads only the footer; in-band -null policies may need a first-batch sample. Flattening a single unnamed -`list` root reads the smallest leaf from each group to establish the -logical row count. -`max_rows` stops that preparation at the requested prefix. `cache_dir` retains -converted groups and the row map for later opens; without it, an in-memory LRU -is used. `refresh()` reopens a changed source and invalidates old views; it -returns without rebuilding the table when a disk-cached source is unchanged. `lazy=False` -calls the eager `CTable.from_parquet()` importer. -The disk cache uses a readable source directory such as -`parquet-cache/readings.parquet--/.b2d/`. Each accessed -physical field and row group becomes a native CTable directory inside that -generation. Opening the generation path with `blosc2.open()` returns the whole -logical remote table and fetches missing groups from the source. -The generation also retains Parquet discovery metadata. A warm open restores -the schema and row-group map locally, without contacting the source. A small -local marker index locates the active source revision. Like other remote table -caches, it assumes that revision is immutable until `refresh()` is called. -The Arrow reader is created only when an uncached group is requested. -Cached row groups open directly from their native `.b2d` directories; warm -reads do not copy complete groups into memory. -Reads against the shared seekable Parquet handle are serialized; increasing -`max_concurrency` does not make one file's row-group reads parallel. +Reads fetch a Parquet field one row group at a time. Selecting one column avoids +unrelated columns; reading a complete row may fetch every column in its group. +`cache_dir` retains converted data and table metadata across opens, so warm +opens and cached reads need no connection to the source. Cached sources are +assumed unchanged until `refresh()` is called. Use `lazy=False` to import the +whole table eagerly. + HTTP servers must honor byte-range requests; fsspec raises a range-request error for servers that only return complete files. Download the file and import it locally when range access is unavailable. -Parquet disk caches require source size and a version marker. HTTP sources -can use an ETag, modification time, or Backblaze B2 file ID. -`table.save("reference.b2z")` creates a portable archive with the retained -groups; `blosc2.open("reference.b2z")` reuses those groups and fetches uncached -ones on demand. The earlier `reference.b2nd` format remains readable. -`table.save("reference.b2nd")` includes already converted groups and the row -map; use `include_cache=False` for a small cold reference. `blosc2.open()` -reopens a reference when the source marker still matches. Runtime credentials -and reader objects are omitted. Reopen a protected source with -`blosc2.RemoteCTable.open_reference("reference.b2nd", storage_options={...})`. -`shared_cache=True` permits separate processes to reuse a `cache_dir`; it -requires a disk cache and serializes cache lookup and publication with a file -lock. Every process sharing that directory should use the same source and -conversion options. `RemoteCTable.with_sparse_cache(url, cache_dir)` uses the -same Parquet cache for a single `.parquet` source. -The `traffic` counter reports file-handle reads and bytes returned; -for an HTTP or object-store driver, its own buffering can change actual wire -traffic. +`table.save("reference.b2z")` creates a portable archive with data already in +the cache. Opening it reuses that data and fetches missing groups on demand. +Use `include_cache=False` to save only the table metadata. Older `.b2nd` +references remain readable. + +Use `shared_cache=True` when separate processes need to share one `cache_dir`. +They should use the same source and conversion options. `blosc2.open()` dispatches uncached local B2Z table archives to `CTable`, and remote B2Z archives and selected local or remote PyTables tables to diff --git a/doc/reference/remotectable.rst b/doc/reference/remotectable.rst index 22ea13dda..b756a6b55 100644 --- a/doc/reference/remotectable.rst +++ b/doc/reference/remotectable.rst @@ -4,8 +4,8 @@ RemoteCTable ============ ``RemoteCTable`` is a read-only :class:`blosc2.CTable` backed by a remote B2Z -archive or a local or remote PyTables/HDF5 table. Fixed-width, shaped, nullable, -UTF-8, batch-backed variable-length, +archive, a local or remote PyTables/HDF5 table, or a Parquet file. Fixed-width, +shaped, nullable, UTF-8, batch-backed variable-length, batch-backed list, struct/object, and dictionary columns are fetched on demand. Open local PyTables tables through :func:`blosc2.open` with ``path=`` or a ``::table`` selector. @@ -74,6 +74,23 @@ peak RAM. for advanced attachment with manifests or seed carriers, with the same default budget and shared-cache implementation. +Parquet cache details +--------------------- + +Parquet ``cache_dir`` stores each accessed physical field and row group as a +native CTable directory beneath ``.parquet--/.b2d/``. +The generation manifest retains the table schema and row-group boundaries; a +small index in ``cache_dir`` identifies its source revision. Warm opens use +these local files without contacting the source. Cached sources are assumed +immutable until ``refresh()`` is called. The generation path itself can be +opened with :func:`blosc2.open` to recover the complete logical table and fetch +uncached groups on demand. + +HTTP Parquet sources need byte-range support. A persistent cache also needs a +source size and version marker, such as an ETag, modification time, or +Backblaze B2 file ID. The ``traffic`` counter measures file-handle reads; +transport buffering may make actual HTTP transfer different. + See :doc:`Working with Remote Tables <../guides/remote_tables>` for column access, filtering, buffering, reference saving, and materialization examples. From d57805a898d88c81cda7b504286c18cbe17753c7 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 14:02:12 +0200 Subject: [PATCH 07/19] Fix Parquet cache locking and archive handling --- src/blosc2/remote_parquet.py | 47 ++++++++++--------- tests/ctable/test_remote_parquet.py | 72 ++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 23 deletions(-) diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index decfb363b..b2f55359f 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -11,7 +11,7 @@ import uuid import zipfile from collections import OrderedDict -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext from dataclasses import asdict, fields from enum import Enum from pathlib import Path @@ -362,8 +362,9 @@ def __init__( self._cache_lru = OrderedDict() self.cache_coordinator = CacheCoordinator(max_cache_bytes) if disk is not None else None if self.cache_coordinator is not None: - self._sync_evictions() - self.cache_coordinator.register(self) + with _disk_guard(self.disk): + self._sync_evictions() + self.cache_coordinator.register(self) self._cache_manager = None self.row_ends = np.cumsum( [self.arrow_metadata.row_group(i).num_rows for i in range(self.arrow_metadata.num_row_groups)] @@ -386,11 +387,15 @@ def release(self): if self._cache_manager is not None: self._cache_manager.cleanup() + @contextmanager def group(self, number, physical): - key = number, physical - with self.lock: - with _disk_guard(self.disk): - return self._group_locked(number, physical, key) + with self.lock, _disk_guard(self.disk): + table = self._group_locked(number, physical, (number, physical)) + try: + yield table + finally: + if self.cache_policy is not blosc2.CachePolicy.MEMORY: + table.close() def _sync_evictions(self): if self.cache_dir is None: @@ -554,15 +559,11 @@ def dictionary(self): values = [] seen = set() for group in range(len(self.storage.row_ends)): - table = self.storage._owner.group(group, self.storage.physical[self.name]) - try: + with self.storage._owner.group(group, self.storage.physical[self.name]) as table: for value in table._cols[self.name].dictionary: if value not in seen: values.append(value) seen.add(value) - finally: - if self.storage._owner.cache_policy is not blosc2.CachePolicy.MEMORY: - table.close() self._dictionary = values return values @@ -599,16 +600,12 @@ def __getitem__(self, key): selected = np.flatnonzero(groups == group) start = 0 if group == 0 else int(self.storage.row_ends[group - 1]) local = positions[selected] - start - table = self.storage._owner.group(int(group), self.storage.physical[self.name]) - try: + with self.storage._owner.group(int(group), self.storage.physical[self.name]) as table: if self.mask: mask = table._null_mask(self.name) part = np.ones(len(local), dtype=bool) if mask is None else mask[local] else: part = table._cols[self.name][local] - finally: - if self.storage._owner.cache_policy is not blosc2.CachePolicy.MEMORY: - table.close() for target, value in zip(selected, part, strict=True): values[int(target)] = value result = ( @@ -779,7 +776,8 @@ def __new__( # noqa: C901 ) handle = None try: - discovery = disk.load()["metadata"].get("parquet_discovery") if disk is not None else None + with _disk_guard(disk): + discovery = disk.load()["metadata"].get("parquet_discovery") if disk is not None else None if discovery is not None: try: schema = schema_from_dict(discovery["schema"]) @@ -1103,7 +1101,12 @@ def _save_archive(self, destination, *, include_cache=True, overwrite=False, mut raise ValueError("destination cannot be inside the live cache") if owner.source_marker is None: owner.source_marker = _source_marker(owner.urlpath, owner.reopen_kwargs["storage_options"]) - with owner.lock, _disk_guard(owner.disk), tempfile.TemporaryDirectory() as temporary: + destination.parent.mkdir(parents=True, exist_ok=True) + with ( + owner.lock, + _disk_guard(owner.disk), + tempfile.TemporaryDirectory(dir=destination.parent) as temporary, + ): if owner.disk is None: disk, root = _disk_cache_path( owner.urlpath, @@ -1136,7 +1139,6 @@ def _save_archive(self, destination, *, include_cache=True, overwrite=False, mut for file in root.rglob("*"): if file.is_file() and not file.name.startswith(".") and file != root / "embed.b2e": archive.write(file, f"{root.name}/{file.relative_to(root)}") - destination.parent.mkdir(parents=True, exist_ok=True) os.replace(staged, destination) return str(destination) @@ -1169,14 +1171,14 @@ def _open_cache_artifact(cls, path, *, mode="r", storage_options=None, source_ur _refresh_marker=manifest["metadata"]["source_marker"], **options, ) - if table._remote_storage()._owner.disk.path != path.parent: + if table._remote_storage()._owner.cache_dir != path: table.close() raise ValueError("Parquet cache source or options no longer match this generation") return table @classmethod def _open_archive_artifact(cls, path, *, mode="r", storage_options=None, source_url=None): - from blosc2.remote_store_cache import StoreDiskCache + from blosc2.remote_store_cache import StoreDiskCache, validate_generation manager = tempfile.TemporaryDirectory(prefix="parquet-reference-") try: @@ -1186,6 +1188,7 @@ def _open_archive_artifact(cls, path, *, mode="r", storage_options=None, source_ source = manifest["source"] if source.get("kind") != "parquet": raise ValueError("Cache archive is not a Parquet table") + validate_generation(manifest["generation"]) generation = f"{manifest['generation']}.b2d" cache_root = StoreDiskCache.path_for(manager.name, source) cache_root.mkdir(parents=True) diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index f17e2cc21..b9a5036bb 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -569,12 +569,20 @@ def test_existing_cache_adds_local_marker_index(tmp_path, monkeypatch): @pytest.mark.parametrize("disk", [False, True]) -def test_cache_archive_reuses_warm_group_and_fetches_cold_group(tmp_path, disk): +def test_cache_archive_reuses_warm_group_and_fetches_cold_group(tmp_path, disk, monkeypatch): path = tmp_path / "source.parquet" pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) options = {"cache_dir": tmp_path / "cache"} if disk else {} artifact = tmp_path / "reference.b2z" cold_artifact = tmp_path / "cold-reference.b2z" + replace = os.replace + + def checked_replace(source, destination): + if destination in (artifact, cold_artifact): + assert source.parent.parent == destination.parent + return replace(source, destination) + + monkeypatch.setattr(remote_parquet.os, "replace", checked_replace) with blosc2.open(path, **options) as original: assert original["x"][0] == 1 original.save(artifact) @@ -856,3 +864,65 @@ def do_GET(self): server.shutdown() server.server_close() thread.join() + + +def test_shared_warm_read_holds_cache_lock(tmp_path, monkeypatch): + from contextlib import contextmanager + + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, cache_dir=tmp_path / "cache", shared_cache=True) as remote: + assert remote["x"][0] == 1 + owner = remote._remote_storage()._owner + guard = owner.disk.guard + locked = False + + @contextmanager + def tracked_guard(): + nonlocal locked + with guard(): + locked = True + try: + yield + finally: + locked = False + + monkeypatch.setattr(owner.disk, "guard", tracked_guard) + original_open = CTable.open + + def checked_open(*args, **kwargs): + table = original_open(*args, **kwargs) + column = table._cols["x"] + + class CheckedColumn: + def __getitem__(self, key): + assert locked, "shared cache must remain locked during lazy reads" + return column[key] + + table._cols["x"] = CheckedColumn() + return table + + monkeypatch.setattr(CTable, "open", checked_open) + assert remote["x"][0] == 1 + assert not locked + + +def test_archive_rejects_invalid_generation(tmp_path): + import zipfile + + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1]}), path) + archive_path = tmp_path / "reference.b2z" + with blosc2.open(path) as remote: + remote.save(archive_path) + with zipfile.ZipFile(archive_path) as archive: + carrier = blosc2.schunk_from_cframe(archive.read("embed.b2e")) + manifest = carrier.vlmeta["b2remote_manifest"] + manifest["generation"] = str(tmp_path / "escaped") + carrier = blosc2.SChunk(meta={"b2tree": {"version": 1}, "b2remote_store": {"version": 1}}) + carrier.vlmeta["b2remote_manifest"] = manifest + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("embed.b2e", carrier.to_cframe()) + with pytest.raises(ValueError, match="generation"): + blosc2.open(archive_path) + assert not (tmp_path / "escaped.b2d").exists() From bb8083eb7bc93f3f143e7994242a0b38e7501ea6 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 14:11:35 +0200 Subject: [PATCH 08/19] Skip Parquet tests without optional dependencies --- tests/ctable/test_remote_parquet.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index b9a5036bb..fea72ff94 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -9,10 +9,7 @@ from decimal import Decimal from email.utils import formatdate -import fsspec import numpy as np -import pyarrow as pa -import pyarrow.parquet as pq import pytest import blosc2 @@ -20,6 +17,10 @@ from blosc2.ctable import CTable from blosc2.schema_compiler import schema_to_dict +fsspec = pytest.importorskip("fsspec") +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + @pytest.mark.parametrize( "source", From b579301849ae10ed7b5bff026ee79f645d25185a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 14:19:46 +0200 Subject: [PATCH 09/19] Address Parquet review findings --- src/blosc2/ctable_remote_read.py | 43 ++++++++++----------- src/blosc2/remote_parquet.py | 13 +++++-- src/blosc2/schunk.py | 4 +- tests/ctable/test_remote_parquet.py | 58 ++++++++++++++++++++++++++++- 4 files changed, 91 insertions(+), 27 deletions(-) diff --git a/src/blosc2/ctable_remote_read.py b/src/blosc2/ctable_remote_read.py index e38752cf3..2d0387f97 100644 --- a/src/blosc2/ctable_remote_read.py +++ b/src/blosc2/ctable_remote_read.py @@ -346,27 +346,28 @@ def column_values(table, names, positions, *, null_masks=None): # noqa: C901 storage = table._remote_read_storage() if storage._owner.format == "parquet": - storage._check_open() - result = {} - for name in names: - result[name] = ( - table._cols[name][positions] - if name in table.col_names - else table._fetch_col_at_positions_uncached(name, positions) - ) - if null_masks is not None and name in table.col_names: - mask = table._null_mask(name) - if mask is not None: - null_masks[name] = ~mask[positions] - elif name in table.col_names: - mask = table._null_mask(name) - if mask is not None: - missing = ~mask[positions] - if missing.any(): - result[name] = list(result[name]) - for index in np.flatnonzero(missing): - result[name][index] = None - return result + with storage._owner.lock: + storage._check_open() + result = {} + for name in names: + result[name] = ( + table._cols[name][positions] + if name in table.col_names + else table._fetch_col_at_positions_uncached(name, positions) + ) + if null_masks is not None and name in table.col_names: + mask = table._null_mask(name) + if mask is not None: + null_masks[name] = ~mask[positions] + elif name in table.col_names: + mask = table._null_mask(name) + if mask is not None: + missing = ~mask[positions] + if missing.any(): + result[name] = list(result[name]) + for index in np.flatnonzero(missing): + result[name][index] = None + return result source_columns = _source_columns(table) with storage._owner.lock: storage._check_open() diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index b2f55359f..8d9e31d56 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -516,9 +516,7 @@ def _group_locked(self, number, physical, key): self.cache[key] = table, size self.cache_bytes += size while ( - self.max_cache_bytes is not None - and self.cache_bytes > self.max_cache_bytes - and len(self.cache) > 1 + self.max_cache_bytes is not None and self.cache_bytes > self.max_cache_bytes and self.cache ): _, (_, removed) = self.cache.popitem(last=False) self.cache_bytes -= removed @@ -554,6 +552,11 @@ def flush(self): @property def dictionary(self): + with self.storage._owner.lock: + self.storage._check_open() + return self._dictionary_locked() + + def _dictionary_locked(self): if self._dictionary is not None: return self._dictionary values = [] @@ -579,6 +582,10 @@ def __getitem__(self, key): return Codes() def __getitem__(self, key): + with self.storage._owner.lock: + return self._getitem_locked(key) + + def _getitem_locked(self, key): self.storage._check_open() if isinstance(key, tuple) and len(key) == 1: key = key[0] diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index dc887c815..bdc34484c 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2983,11 +2983,13 @@ def open( # noqa: C901 from blosc2.remote_parquet import RemoteParquetCTable kwargs.pop("source_format", None) + lazy = kwargs.pop("lazy", True) + if shared_cache and lazy is False: + raise TypeError("shared_cache requires lazy Parquet access") if shared_cache: if kwargs.get("cache_dir") is None: raise ValueError("shared_cache=True requires cache_dir") kwargs["shared_cache"] = True - lazy = kwargs.pop("lazy", True) if lazy is False: from blosc2.core import fsspec_open diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index fea72ff94..a4562a7db 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -6,6 +6,7 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from decimal import Decimal from email.utils import formatdate @@ -293,6 +294,8 @@ def test_invalid_source_options_fail_clearly(tmp_path): blosc2.open(path, lazy=False, cache_dir=tmp_path / "cache") with pytest.raises(ValueError, match="requires cache_dir"): blosc2.open(path, shared_cache=True) + with pytest.raises(TypeError, match="shared_cache requires lazy"): + blosc2.open(path, lazy=False, shared_cache=True, cache_dir=tmp_path / "cache") with pytest.raises(ValueError, match="memory_map"): blosc2.open(path, parquet_options={"memory_map": True}) with pytest.raises(ValueError, match="conflicts"): @@ -868,8 +871,6 @@ def do_GET(self): def test_shared_warm_read_holds_cache_lock(tmp_path, monkeypatch): - from contextlib import contextmanager - path = tmp_path / "source.parquet" pq.write_table(pa.table({"x": [1, 2]}), path) with blosc2.open(path, cache_dir=tmp_path / "cache", shared_cache=True) as remote: @@ -927,3 +928,56 @@ def test_archive_rejects_invalid_generation(tmp_path): with pytest.raises(ValueError, match="generation"): blosc2.open(archive_path) assert not (tmp_path / "escaped.b2d").exists() + + +@pytest.mark.parametrize("whole_row", [False, True]) +def test_refresh_waits_for_parquet_read(tmp_path, monkeypatch, whole_row): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) + with blosc2.open(path) as remote: + owner = remote._remote_storage()._owner + original_group = owner.group + entered = threading.Event() + resume = threading.Event() + refresh_started = threading.Event() + refresh_done = threading.Event() + + @contextmanager + def paused_group(number, physical): + with original_group(number, physical) as table: + if number == 0 and physical == "x": + entered.set() + assert resume.wait(5) + yield table + + monkeypatch.setattr(owner, "group", paused_group) + + def refresh(): + refresh_started.set() + remote.refresh() + refresh_done.set() + + with ThreadPoolExecutor(max_workers=2) as pool: + read = pool.submit(lambda: remote[0] if whole_row else remote["x"][:]) + assert entered.wait(5) + update = pool.submit(refresh) + assert refresh_started.wait(5) + assert not refresh_done.wait(0.1) + resume.set() + result = read.result(timeout=5) + update.result(timeout=5) + if whole_row: + assert (result.x, result.y) == (1, 5) + else: + assert result.tolist() == [1, 2, 3, 4] + + +def test_oversized_memory_group_is_not_retained(tmp_path): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, max_cache_bytes=1) as remote: + assert remote["x"][0] == 1 + assert remote.cache_bytes == 0 + before = remote.traffic.requests + assert remote["x"][1] == 2 + assert remote.traffic.requests > before From 2e595b3a5956d0466ef54dd03eaae6f3f1e9da9a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 14:31:10 +0200 Subject: [PATCH 10/19] Run Parquet tests in native CI --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 13a2ce117..2ffc2c967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,8 @@ test = [ # feature silently skips in CI. memory:// covers the protocol-generic paths, # and is also the configuration most users installing [fsspec] are in. "fsspec; platform_machine != 'wasm32'", + # Exercise Parquet import and lazy remote-table tests in native CI jobs. + "pyarrow; platform_machine != 'wasm32'", # Exercise the optional remote array sources instead of silently skipping # tests/test_zarr_source.py and tests/test_hdf5_source.py. "zarr>=3.0.9; platform_machine != 'wasm32'", From 4ee1453cb7228fe742bbb0b65d2ed1a227718a85 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 14:44:49 +0200 Subject: [PATCH 11/19] Handle Windows paths in Parquet caches --- src/blosc2/core.py | 7 +++++-- src/blosc2/remote_parquet.py | 28 ++++++++++++++++++---------- tests/ctable/test_remote_parquet.py | 5 +++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 67b240def..477e3ca6f 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -811,8 +811,11 @@ def cache_path_component(value: str) -> str: def cache_directory_name(urlpath: str, identity: bytes) -> str: """A recognizable source basename and a 48-bit cache identity.""" - parsed = urllib.parse.urlsplit(urlpath) - name = pathlib.PurePosixPath(parsed.path.rstrip("/")).name or parsed.hostname or "remote" + if pathlib.PureWindowsPath(urlpath).drive and not is_fsspec_url(urlpath): + name = pathlib.PureWindowsPath(urlpath).name + else: + parsed = urllib.parse.urlsplit(urlpath) + name = pathlib.PurePosixPath(parsed.path.rstrip("/")).name or parsed.hostname or "remote" name = cache_path_component(urllib.parse.unquote(name)) return name + "--" + hashlib.sha256(identity).hexdigest()[:12] diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index 8d9e31d56..0a7104700 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -20,6 +20,7 @@ import numpy as np import blosc2 +from blosc2.core import is_fsspec_url, normalize_urlpath from blosc2.ctable import CTable, NullPolicy, get_null_policy, null_policy from blosc2.ctable_storage import RemoteTableStorage, _AllValidRows, split_field_path from blosc2.proxy import CacheCoordinator @@ -205,10 +206,18 @@ def _open_source_handle(urlpath, storage_options, marker, traffic): return _CountingHandle(fs.open(path, "rb", **kwargs), traffic) +def _source_url(urlpath): + urlpath = str(normalize_urlpath(urlpath)) + if not is_fsspec_url(urlpath): + return os.path.abspath(urlpath) + url = urlsplit(urlpath) + return urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")) + + def _cache_marker_index(urlpath, storage_options, options, cache_dir): import pyarrow - if not urlsplit(urlpath).scheme: + if not is_fsspec_url(urlpath): urlpath = os.path.abspath(urlpath) identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, options) digest = hashlib.sha256(repr(identity).encode()).hexdigest() @@ -249,14 +258,13 @@ def _disk_cache_path(urlpath, storage_options, options, cache_dir, marker, *, sh if "size" not in marker or len(marker) < 2: raise ValueError("A persistent Parquet cache requires source size and a version marker") - if not urlsplit(urlpath).scheme: + if not is_fsspec_url(urlpath): urlpath = os.path.abspath(urlpath) identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, marker, options) digest = hashlib.sha256(repr(identity).encode()).hexdigest() - url = urlsplit(urlpath) source = { "kind": "parquet", - "urlpath": urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")), + "urlpath": _source_url(urlpath), "identity": digest, } disk = (SharedStoreCache if shared else StoreDiskCache)(cache_dir, source) @@ -718,6 +726,7 @@ def __new__( # noqa: C901 import pyarrow as pa import pyarrow.parquet as pq + urlpath = str(normalize_urlpath(urlpath)) cache_dir = kwargs.pop("cache_dir", None) seed_cache = kwargs.pop("_seed_cache", None) cache_manager = kwargs.pop("_cache_manager", None) @@ -1047,11 +1056,10 @@ def refresh(self): @property def source(self): - url = urlsplit(self._remote_storage()._owner.urlpath) return { "kind": "parquet", "version": 1, - "urlpath": urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")), + "urlpath": _source_url(self._remote_storage()._owner.urlpath), } @property @@ -1099,7 +1107,7 @@ def _save_archive(self, destination, *, include_cache=True, overwrite=False, mut if mutable not in (None, False): raise ValueError("Parquet references are read-only") owner = self._remote_storage()._owner - if urlsplit(owner.urlpath).scheme: + if is_fsspec_url(owner.urlpath): validate_persistable_url(owner.urlpath) destination = Path(destination) if destination.exists() and not overwrite: @@ -1159,7 +1167,7 @@ def _open_cache_artifact(cls, path, *, mode="r", storage_options=None, source_ur source = manifest["source"] if source.get("kind") != "parquet": raise ValueError("Cache is not a Parquet table") - if urlsplit(source["urlpath"]).scheme: + if is_fsspec_url(source["urlpath"]): from blosc2.remote_array import validate_persistable_url validate_persistable_url(source["urlpath"]) @@ -1236,7 +1244,7 @@ def _save_reference(self, destination, *, include_cache=True, overwrite=False, m raise ValueError("Parquet references are read-only") owner = self._remote_storage()._owner url = owner.urlpath - if urlsplit(url).scheme: + if is_fsspec_url(url): validate_persistable_url(url) if owner.source_marker is None: owner.source_marker = _source_marker(url, owner.reopen_kwargs["storage_options"]) @@ -1301,7 +1309,7 @@ def _from_payload(cls, payload, *, storage_options=None, parquet_options=None): options = payload.get("options") if not isinstance(url, str) or not isinstance(marker, dict) or not isinstance(options, dict): raise ValueError("Invalid Parquet reference fields") - if urlsplit(url).scheme: + if is_fsspec_url(url): validate_persistable_url(url) if "storage_options" in options or "cache_dir" in options: raise ValueError("Parquet references cannot contain runtime transport options") diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index a4562a7db..a34f6f550 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -15,6 +15,7 @@ import blosc2 from blosc2 import remote_parquet +from blosc2.core import cache_directory_name from blosc2.ctable import CTable from blosc2.schema_compiler import schema_to_dict @@ -23,6 +24,10 @@ pq = pytest.importorskip("pyarrow.parquet") +def test_windows_cache_directory_uses_source_basename(): + assert cache_directory_name(r"C:\data\source.parquet", b"cache").startswith("source.parquet--") + + @pytest.mark.parametrize( "source", [ From 40af8a5349bba75529ba363d03196e2660cacd8c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 15:07:39 +0200 Subject: [PATCH 12/19] Preserve Parquet source queries and reader options --- src/blosc2/remote_parquet.py | 13 +++++++------ tests/ctable/test_remote_parquet.py | 22 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index 0a7104700..8ff38756f 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -211,7 +211,7 @@ def _source_url(urlpath): if not is_fsspec_url(urlpath): return os.path.abspath(urlpath) url = urlsplit(urlpath) - return urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")) + return urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, url.query, "")) def _cache_marker_index(urlpath, storage_options, options, cache_dir): @@ -267,6 +267,10 @@ def _disk_cache_path(urlpath, storage_options, options, cache_dir, marker, *, sh "urlpath": _source_url(urlpath), "identity": digest, } + if is_fsspec_url(urlpath): + from blosc2.remote_array import validate_persistable_url + + validate_persistable_url(source["urlpath"]) disk = (SharedStoreCache if shared else StoreDiskCache)(cache_dir, source) try: with _disk_guard(disk): @@ -1321,11 +1325,8 @@ def _from_payload(cls, payload, *, storage_options=None, parquet_options=None): raise ValueError("Parquet reference has no null policy") _restore_compression_options(options) saved_reader = options.get("parquet_options") or {} - if parquet_options: - for key, value in parquet_options.items(): - if key in saved_reader and saved_reader[key] != value: - raise ValueError(f"Parquet reader option {key!r} differs from the saved reference") - options["parquet_options"] = {**saved_reader, **parquet_options} + if parquet_options is not None and parquet_options != saved_reader: + raise ValueError("Parquet reader option set differs from the saved reference") retained = payload.get("cache_files", {}) if not isinstance(retained, dict): raise ValueError("Invalid retained Parquet cache") diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index a34f6f550..9ef077d87 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -28,6 +28,21 @@ def test_windows_cache_directory_uses_source_basename(): assert cache_directory_name(r"C:\data\source.parquet", b"cache").startswith("source.parquet--") +def test_source_url_preserves_non_sensitive_query(): + assert ( + remote_parquet._source_url("https://example.com/source.parquet?version=2#section") + == "https://example.com/source.parquet?version=2" + ) + with pytest.raises(ValueError, match="credential-like"): + remote_parquet._disk_cache_path( + "https://example.com/source.parquet?token=secret", + None, + (), + None, + {"size": "1", "etag": "revision"}, + ) + + @pytest.mark.parametrize( "source", [ @@ -378,11 +393,16 @@ def test_reference_reader_options_are_frozen(tmp_path): carrier = tmp_path / "source.b2nd" pq.write_table(pa.table({"name": ["a", "b", "a"]}), path) with blosc2.open(path, parquet_options={"read_dictionary": ["name"]}) as remote: - remote.save(carrier, include_cache=False) + assert remote["name"][0] == "a" + remote.save(carrier, include_cache=True) with blosc2.RemoteCTable.open_reference(carrier) as reopened: assert reopened.to_arrow().column("name").to_pylist() == ["a", "b", "a"] with pytest.raises(ValueError, match="differs"): blosc2.RemoteCTable.open_reference(carrier, parquet_options={"read_dictionary": []}) + with pytest.raises(ValueError, match="differ"): + blosc2.RemoteCTable.open_reference( + carrier, parquet_options={"read_dictionary": ["name"], "coerce_int96_timestamp_unit": "ms"} + ) def test_closed_handle_rejects_cached_column(tmp_path): From c235b210fd563d7d155c40fc0015b429ab8a68b3 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 15:14:16 +0200 Subject: [PATCH 13/19] Redact signed queries from Parquet source descriptions --- src/blosc2/remote_parquet.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index 8ff38756f..e864c4da2 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -1060,10 +1060,18 @@ def refresh(self): @property def source(self): + source_url = _source_url(self._remote_storage()._owner.urlpath) + if is_fsspec_url(source_url): + from blosc2.remote_array import validate_persistable_url + + try: + validate_persistable_url(source_url) + except ValueError: + source_url = source_url.split("?", 1)[0] return { "kind": "parquet", "version": 1, - "urlpath": _source_url(self._remote_storage()._owner.urlpath), + "urlpath": source_url, } @property From fbdd520a797237e81d6a8649c74d74d0a0143a88 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 15:27:10 +0200 Subject: [PATCH 14/19] Count HTTP requests before sending test responses --- tests/ctable/test_remote_parquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index 9ef077d87..8210cef1e 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -809,9 +809,9 @@ def do_GET(self): self.send_header("Accept-Ranges", "bytes") self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) self.end_headers() - self.wfile.write(data) counts["requests"] += 1 counts["bytes"] += len(data) + self.wfile.write(data) try: server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Ranged) From 0fdc62c7de28159c6f71816bae993dd749201ab2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 18:29:05 +0200 Subject: [PATCH 15/19] Consolidate remote table examples and improve writes --- doc/guides/remote_tables.md | 6 + examples/ctable/remote_handling.py | 276 ++++++++++++++++++++++++----- examples/ctable/remote_pytables.py | 193 -------------------- tests/ctable/test_remote_ctable.py | 108 +++++++++++ 4 files changed, 350 insertions(+), 233 deletions(-) delete mode 100644 examples/ctable/remote_pytables.py diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index 3e7c73294..eca6626bf 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -165,6 +165,12 @@ See `examples/ctable/remote_handling.py` for a batched archive writer with nulla multilingual UTF-8 and variable-length strings, a batch-backed list, and a dictionary. It reports ordinary batch cold/warm reads and dictionary code/vocabulary costs separately. +The same script writes and reads Blosc2 with `--blosc2`, PyTables/HDF5 with +`--pytables`, and Parquet with `--parquet`. Without a format flag, it selects the +format from the `.b2z`, `.h5`, or `.parquet` extension of the output path or input URL. +Use `--write FILE` to create a file locally, then pass its uploaded URL to read it remotely. +The Parquet file uses the full Blosc2 example schema, including nullable values, +multilingual strings, lists, and dictionary-encoded regions. ## Refresh a remote table diff --git a/examples/ctable/remote_handling.py b/examples/ctable/remote_handling.py index 71355e385..93e65e475 100644 --- a/examples/ctable/remote_handling.py +++ b/examples/ctable/remote_handling.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Create or access a CTable with fixed-width, UTF-8, batch-backed and dictionary columns.""" +"""Create or access remote-friendly Blosc2, PyTables, and Parquet tables.""" import argparse import pprint @@ -14,6 +14,7 @@ import time from dataclasses import dataclass, fields from pathlib import Path +from urllib.parse import urlsplit import numpy as np @@ -40,15 +41,38 @@ class Reading: FULL_INDEX_UNSUPPORTED = {"message", "tags"} +TABLE_NAME = "readings" def make_notes(ids): notes = np.array(["", "café", "東京の観測", "🌦️ weather improving"], dtype=object)[ids % 4] - notes = np.array( + return np.array( [f"{text} #{i}" if text else "" for i, text in zip(ids, notes, strict=True)], dtype=object ) - notes[ids % 43 == 0] = None - return notes + + +def rich_batches(rows, batch_size): + rng = np.random.default_rng(42) + statuses = np.array(["ok", "warning", "offline", ""], dtype=object) + for start in range(0, rows, batch_size): + ids = np.arange(start, min(start + batch_size, rows), dtype=np.int64) + temperature = rng.normal(18, 10, len(ids)).astype(np.float32) + humidity = rng.integers(0, 101, len(ids), dtype=np.int16) + status = statuses[(ids // 7) % len(statuses)] + regions = np.array(["north", "south", "east", "west"], dtype=object)[ids % 4] + regions[ids % 59 == 0] = None + yield { + "id": ids, + "station_id": (ids % 100).astype(np.int32), + "temperature": np.ma.array(temperature, mask=ids % 17 == 0), + "humidity": np.ma.array(humidity, mask=ids % 29 == 0), + "status": np.ma.array(status, mask=ids % 41 == 0), + "active": ids % 5 != 0, + "note": np.ma.array(make_notes(ids), mask=ids % 43 == 0), + "message": [None if i % 47 == 0 else f"sensor {i}: café 東京" for i in ids], + "tags": [None if i % 53 == 0 else [int(i % 7), int(i % 11)] for i in ids], + "region": regions, + } def write_table(args) -> None: @@ -77,8 +101,6 @@ def write_table(args) -> None: raise ValueError("--full columns must not contain duplicates") output.parent.mkdir(parents=True, exist_ok=True) - rng = np.random.default_rng(42) - statuses = np.array(["ok", "warning", "offline", ""], dtype=object) attrs = { "version": 1, "sampling_interval": 0.5, @@ -96,34 +118,8 @@ def write_table(args) -> None: ) as table: for name, value in attrs.items(): table.attrs[name] = value - for start in range(0, args.rows, args.batch_size): - stop = min(start + args.batch_size, args.rows) - ids = np.arange(start, stop, dtype=np.int64) - temperature = rng.normal(18, 10, len(ids)).astype(np.float32).astype(object) - humidity = rng.integers(0, 101, len(ids), dtype=np.int16).astype(object) - status = statuses[(ids // 7) % len(statuses)].copy() - temperature[ids % 17 == 0] = None - humidity[ids % 29 == 0] = None - status[ids % 41 == 0] = None - messages = [None if i % 47 == 0 else f"sensor {i}: café 東京" for i in ids] - tags = [None if i % 53 == 0 else [int(i % 7), int(i % 11)] for i in ids] - regions = np.array(["north", "south", "east", "west"], dtype=object)[ids % 4] - regions[ids % 59 == 0] = None - table.extend( - { - "id": ids, - "station_id": (ids % 100).astype(np.int32), - "temperature": temperature, - "humidity": humidity, - "status": status, - "active": ids % 5 != 0, - "note": make_notes(ids), - "message": messages, - "tags": tags, - "region": regions, - }, - validate=False, - ) + for batch in rich_batches(args.rows, args.batch_size): + table.extend(batch, validate=False) if indexed_columns: started = time.perf_counter() for name in indexed_columns: @@ -149,6 +145,134 @@ def write_table(args) -> None: print(f"Now upload {output} to your cloud object storage.") +def simple_batches(rows, batch_size): + rng = np.random.default_rng(42) + statuses = np.array(["ok", "warning", "offline", ""]) + notes = np.array(["", "clear", "cloudy", "rain"]) + for start in range(0, rows, batch_size): + ids = np.arange(start, min(start + batch_size, rows), dtype=np.int64) + yield { + "id": ids, + "station_id": (ids % 100).astype(np.int32), + "temperature": rng.normal(18, 10, len(ids)).astype(np.float32), + "humidity": rng.integers(0, 101, len(ids), dtype=np.int16), + "status": statuses[(ids // 7) % len(statuses)], + "active": ids % 5 != 0, + "note": notes[ids % len(notes)], + } + + +def write_pytables(args) -> None: + import tables + + class PyTablesReading(tables.IsDescription): + id = tables.Int64Col(pos=0) + station_id = tables.Int32Col(pos=1) + temperature = tables.Float32Col(pos=2) + humidity = tables.Int16Col(pos=3) + status = tables.StringCol(8, pos=4) + active = tables.BoolCol(pos=5) + note = tables.StringCol(32, pos=6) + + output = args.write + if output.suffix != ".h5": + raise ValueError("output must end in .h5") + if args.rows < 1 or args.batch_size < 1: + raise ValueError("--rows and --batch-size must be positive") + if output.exists() and not args.overwrite: + raise FileExistsError(f"{output} already exists; pass --overwrite to replace it") + indexed_columns = [] + if args.full is not None: + indexed_columns = ( + list(PyTablesReading.columns) + if args.full == "*" + else [name.strip() for name in args.full.split(",")] + ) + unknown = set(indexed_columns) - set(PyTablesReading.columns) + if not all(indexed_columns) or unknown: + raise ValueError(f"invalid --full columns: {', '.join(sorted(unknown)) or args.full!r}") + if len(indexed_columns) != len(set(indexed_columns)): + raise ValueError("--full columns must not contain duplicates") + + output.parent.mkdir(parents=True, exist_ok=True) + with tables.open_file(output, mode="w") as h5file: + filters = tables.Filters(complevel=5, complib="blosc2:zstd", shuffle=True) + table = h5file.create_table( + "/", + TABLE_NAME, + PyTablesReading, + title="Synthetic weather-station readings", + filters=filters, + expectedrows=args.rows, + ) + table.attrs.version = 1 + table.attrs.sampling_interval = 0.5 + table.attrs.description = "Synthetic weather-station readings" + + for batch in simple_batches(args.rows, args.batch_size): + data = np.empty(len(batch["id"]), dtype=table.dtype) + for name, values in batch.items(): + data[name] = values + table.append(data) + table.flush() + + if indexed_columns: + started = time.perf_counter() + for name in indexed_columns: + getattr(table.cols, name).create_csindex(filters=filters) + print(f"Created FULL (CSI) indexes in {time.perf_counter() - started:.2f} s") + + with tables.open_file(output) as h5file: + table = h5file.root.readings + assert table.nrows == args.rows + assert all(getattr(table.cols, name).index.is_csi for name in indexed_columns) + + print(f"Created {output} ({output.stat().st_size / 1_000_000:.1f} MB, {args.rows:,} rows)") + print(f"FULL (CSI) indexes: {', '.join(indexed_columns) if indexed_columns else 'none'}") + print(f"Now upload {output} to your cloud object storage.") + + +def write_parquet(args) -> None: + import pyarrow as pa + import pyarrow.parquet as pq + + output = args.write + if output.suffix != ".parquet": + raise ValueError("output must end in .parquet") + if args.rows < 1 or args.batch_size < 1: + raise ValueError("--rows and --batch-size must be positive") + if output.exists() and not args.overwrite: + raise FileExistsError(f"{output} already exists; pass --overwrite to replace it") + + output.parent.mkdir(parents=True, exist_ok=True) + schema = pa.schema( + [ + ("id", pa.int64()), + ("station_id", pa.int32()), + ("temperature", pa.float32()), + ("humidity", pa.int16()), + ("status", pa.string()), + ("active", pa.bool_()), + ("note", pa.large_string()), + ("message", pa.large_string()), + ("tags", pa.list_(pa.int16())), + ("region", pa.dictionary(pa.int32(), pa.string())), + ] + ) + with pq.ParquetWriter( + output, + schema, + compression="zstd", + use_dictionary=["station_id", "humidity", "status", "tags.list.element", "region"], + ) as writer: + for batch in rich_batches(args.rows, args.batch_size): + writer.write_table(pa.table(batch, schema=schema), row_group_size=args.batch_size) + + assert pq.read_metadata(output).num_rows == args.rows + print(f"Created {output} ({output.stat().st_size / 1_000_000:.1f} MB, {args.rows:,} rows)") + print(f"Now upload {output} to your cloud object storage.") + + def access_table(args) -> None: storage_options = {} if args.url.startswith("s3://"): @@ -318,12 +442,71 @@ def access_table(args) -> None: print(f"Retained cache: {table.cache_bytes / 1024:8.2f} KB") +def access_external_table(args) -> None: + storage_options = None + if args.url.startswith("s3://"): + storage_options = { + "profile": args.profile, + "client_kwargs": {"endpoint_url": args.endpoint_url}, + } + + cache_options = {"cache_dir": args.cache_dir} if args.cache_dir is not None else {} + print(f"Accessing: {args.url}{f'::{TABLE_NAME}' if args.pytables else ''}") + started = time.perf_counter() + if args.pytables: + table = blosc2.RemoteCTable( + args.url, dataset=TABLE_NAME, storage_options=storage_options, **cache_options + ) + else: + table = blosc2.open( + args.url, source_format="parquet", storage_options=storage_options, **cache_options + ) + with table: + metadata_time = time.perf_counter() - started + metadata_bytes = table.traffic.nbytes + metadata_requests = table.traffic.requests + metadata = { + "type": type(table).__name__, + "rows": table.nrows, + "columns": table.col_names, + "schema": table.schema_dict(), + "attrs": dict(table.attrs), + "indexes": sorted(table._get_index_catalog()), + } + print(f"\n[Format: {'PyTables/HDF5' if args.pytables else 'Parquet'}]") + for name, value in metadata.items(): + rendered = pprint.pformat(value) if isinstance(value, (dict, list)) else value + print(f"{name:<9}: {rendered}") + + sample_start = max(0, table.nrows // 2 - 2) + sample_stop = min(sample_start + 5, table.nrows) + print(f"\nSample rows [{sample_start}:{sample_stop}]:") + print(table[sample_start:sample_stop]) + + started = time.perf_counter() + ids = table.where("(station_id == 42) & active").id[:5] + query_time = time.perf_counter() - started + print("\nQuery: (station_id == 42) & active") + print(f"first ids: {ids}") + print( + f"metadata: {metadata_time * 1000:.1f} ms, " + f"{metadata_requests} requests, {metadata_bytes / 1024:.2f} KiB" + ) + print( + f"query: {query_time * 1000:.1f} ms, " + f"{table.traffic.requests - metadata_requests} requests, " + f"{(table.traffic.nbytes - metadata_bytes) / 1024:.2f} KiB" + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "url", nargs="?", help="Local .b2z CTable path or remote URL (s3://, http://, https://)" - ) - parser.add_argument("--write", type=Path, metavar="FILE", help="Create a local .b2z CTable instead") + parser.add_argument("url", nargs="?", help="Local table path or remote URL (s3://, http://, https://)") + parser.add_argument("--write", type=Path, metavar="FILE", help="Create a local table") + formats = parser.add_mutually_exclusive_group() + formats.add_argument("--blosc2", action="store_true", help="Use Blosc2 CTable (.b2z)") + formats.add_argument("--pytables", action="store_true", help="Use PyTables/HDF5 (.h5)") + formats.add_argument("--parquet", action="store_true", help="Use Parquet (.parquet)") parser.add_argument( "--full", nargs="?", @@ -344,12 +527,25 @@ def main() -> int: if args.write is not None and args.url is not None: parser.error("URL cannot be combined with --write") if args.write is None and args.url is None: - parser.error("provide a local path or remote URL, or use --write FILE.b2z") + parser.error("provide a local path or remote URL, or use --write FILE") + if not (args.blosc2 or args.pytables or args.parquet): + source = args.write or Path(urlsplit(args.url).path or urlsplit(args.url).netloc) + format_name = {".b2z": "blosc2", ".h5": "pytables", ".parquet": "parquet"}.get(source.suffix) + if format_name is None: + parser.error("cannot infer format; use .b2z, .h5, .parquet, or a format flag") + setattr(args, format_name, True) if args.full is not None and args.write is None: parser.error("--full requires --write") + if args.full is not None and args.parquet: + parser.error("--full is not supported for Parquet") try: - write_table(args) if args.write is not None else access_table(args) + if args.write is not None: + (write_pytables if args.pytables else write_parquet if args.parquet else write_table)(args) + elif args.pytables or args.parquet: + access_external_table(args) + else: + access_table(args) except Exception as exc: print(f"Error: {exc}", file=sys.stderr) return 1 diff --git a/examples/ctable/remote_pytables.py b/examples/ctable/remote_pytables.py deleted file mode 100644 index 7df42b2e5..000000000 --- a/examples/ctable/remote_pytables.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -####################################################################### -# Copyright (c) 2019-present, Blosc Development Team -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -####################################################################### - -"""Create a PyTables table locally or access it remotely through RemoteCTable.""" - -import argparse -import pprint -import sys -import time -from pathlib import Path - -import numpy as np -import tables - -import blosc2 - -DEFAULT_PROFILE = "blosc2" -DEFAULT_ENDPOINT_URL = "https://s3.us-west-001.backblazeb2.com" -TABLE_NAME = "readings" - - -class Reading(tables.IsDescription): - id = tables.Int64Col(pos=0) - station_id = tables.Int32Col(pos=1) - temperature = tables.Float32Col(pos=2) - humidity = tables.Int16Col(pos=3) - status = tables.StringCol(8, pos=4) - active = tables.BoolCol(pos=5) - note = tables.StringCol(32, pos=6) - - -def write_table(args) -> None: - output = args.write - if output.suffix != ".h5": - raise ValueError("output must end in .h5") - if args.rows < 1 or args.batch_size < 1: - raise ValueError("--rows and --batch-size must be positive") - if output.exists() and not args.overwrite: - raise FileExistsError(f"{output} already exists; pass --overwrite to replace it") - indexed_columns = [] - if args.full is not None: - indexed_columns = ( - list(Reading.columns) if args.full == "*" else [name.strip() for name in args.full.split(",")] - ) - unknown = set(indexed_columns) - set(Reading.columns) - if not all(indexed_columns) or unknown: - raise ValueError(f"invalid --full columns: {', '.join(sorted(unknown)) or args.full!r}") - if len(indexed_columns) != len(set(indexed_columns)): - raise ValueError("--full columns must not contain duplicates") - - output.parent.mkdir(parents=True, exist_ok=True) - rng = np.random.default_rng(42) - statuses = np.array([b"ok", b"warning", b"offline", b""], dtype="S8") - notes = np.array([b"", b"clear", b"cloudy", b"rain"], dtype="S32") - - with tables.open_file(output, mode="w") as h5file: - filters = tables.Filters(complevel=5, complib="blosc2:zstd", shuffle=True) - table = h5file.create_table( - "/", - TABLE_NAME, - Reading, - title="Synthetic weather-station readings", - filters=filters, - expectedrows=args.rows, - ) - table.attrs.version = 1 - table.attrs.sampling_interval = 0.5 - table.attrs.description = "Synthetic weather-station readings" - - for start in range(0, args.rows, args.batch_size): - stop = min(start + args.batch_size, args.rows) - ids = np.arange(start, stop, dtype=np.int64) - data = np.empty(len(ids), dtype=table.dtype) - data["id"] = ids - data["station_id"] = ids % 100 - data["temperature"] = rng.normal(18, 10, len(ids)).astype(np.float32) - data["humidity"] = rng.integers(0, 101, len(ids), dtype=np.int16) - data["status"] = statuses[(ids // 7) % len(statuses)] - data["active"] = ids % 5 != 0 - data["note"] = notes[ids % len(notes)] - table.append(data) - table.flush() - - if indexed_columns: - started = time.perf_counter() - for name in indexed_columns: - getattr(table.cols, name).create_csindex(filters=filters) - print(f"Created FULL (CSI) indexes in {time.perf_counter() - started:.2f} s") - - with tables.open_file(output) as h5file: - table = h5file.root.readings - assert table.nrows == args.rows - assert all(getattr(table.cols, name).index.is_csi for name in indexed_columns) - - print(f"Created {output} ({output.stat().st_size / 1_000_000:.1f} MB, {args.rows:,} rows)") - print(f"FULL (CSI) indexes: {', '.join(indexed_columns) if indexed_columns else 'none'}") - print(f"Now upload {output} to your cloud object storage.") - - -def access_table(args) -> None: - storage_options = None - if args.url.startswith("s3://"): - storage_options = { - "profile": args.profile, - "client_kwargs": {"endpoint_url": args.endpoint_url}, - } - - print(f"Accessing: {args.url}::{TABLE_NAME}") - started = time.perf_counter() - cache_options = {"cache_dir": args.cache_dir} if args.cache_dir is not None else {} - with blosc2.RemoteCTable( - args.url, dataset=TABLE_NAME, storage_options=storage_options, **cache_options - ) as table: - metadata_time = time.perf_counter() - started - metadata_bytes = table.traffic.nbytes - metadata_requests = table.traffic.requests - metadata = { - "type": type(table).__name__, - "rows": table.nrows, - "columns": table.col_names, - "schema": table.schema_dict(), - "attrs": dict(table.attrs), - "indexes": sorted(table._get_index_catalog()), - } - print("\n[Format: PyTables/HDF5]") - for name, value in metadata.items(): - rendered = pprint.pformat(value) if isinstance(value, (dict, list)) else value - print(f"{name:<9}: {rendered}") - - sample_start = max(0, table.nrows // 2 - 2) - sample_stop = min(sample_start + 5, table.nrows) - print(f"\nSample rows [{sample_start}:{sample_stop}]:") - print(table[sample_start:sample_stop]) - - started = time.perf_counter() - ids = table.where("(station_id == 42) & active").id[:5] - query_time = time.perf_counter() - started - print("\nQuery: (station_id == 42) & active") - print(f"first ids: {ids}") - print( - f"metadata: {metadata_time * 1000:.1f} ms, " - f"{metadata_requests} requests, {metadata_bytes / 1024:.2f} KiB" - ) - print( - f"query: {query_time * 1000:.1f} ms, " - f"{table.traffic.requests - metadata_requests} requests, " - f"{(table.traffic.nbytes - metadata_bytes) / 1024:.2f} KiB" - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("url", nargs="?", help="Remote .h5 URL (s3://, http://, or https://)") - parser.add_argument("--write", type=Path, metavar="FILE", help="Create a local .h5 table instead") - parser.add_argument( - "--full", - nargs="?", - const="*", - metavar="COL1,COL2", - help="Create FULL (CSI) indexes for every column, or only the comma-separated columns", - ) - parser.add_argument( - "--cache-dir", type=Path, metavar="DIR", help="Persist remote data in DIR (default: memory)" - ) - parser.add_argument("--rows", type=int, default=1_000_000) - parser.add_argument("--batch-size", type=int, default=100_000) - parser.add_argument("--overwrite", action="store_true") - parser.add_argument("--profile", default=DEFAULT_PROFILE) - parser.add_argument("--endpoint-url", default=DEFAULT_ENDPOINT_URL) - args = parser.parse_args() - - if args.write is not None and args.url is not None: - parser.error("URL cannot be combined with --write") - if args.write is None and args.url is None: - parser.error("provide a remote URL or use --write FILE.h5") - if args.full is not None and args.write is None: - parser.error("--full requires --write") - - try: - write_table(args) if args.write is not None else access_table(args) - except Exception as exc: - print(f"Error: {exc}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/ctable/test_remote_ctable.py b/tests/ctable/test_remote_ctable.py index 1d6218067..e6cd3d173 100644 --- a/tests/ctable/test_remote_ctable.py +++ b/tests/ctable/test_remote_ctable.py @@ -689,6 +689,8 @@ def test_remote_example_batch_columns(tmp_path, capsys): assert local["message"][47] is None assert local["tags"][53] is None assert local["region"][59] is None + for name, period in (("temperature", 17), ("humidity", 29), ("status", 41), ("note", 43)): + np.testing.assert_array_equal(np.flatnonzero(local[name].is_null()), np.arange(0, 100, period)) url = f"memory://{tmp_path.name}-example-batches.b2z" fsspec.filesystem("memory").pipe(url, path.read_bytes()) @@ -700,6 +702,112 @@ def test_remote_example_batch_columns(tmp_path, capsys): assert "Dictionary costs (codes first, then full vocabulary on first decode)" in output +@pytest.mark.parametrize(("format_flag", "suffix"), [("pytables", ".h5"), ("parquet", ".parquet")]) +@pytest.mark.parametrize("explicit", [False, True]) +def test_remote_example_external_formats(tmp_path, capsys, monkeypatch, format_flag, suffix, explicit): + import runpy + import sys + from pathlib import Path + + pytest.importorskip("tables" if format_flag == "pytables" else "pyarrow") + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + path = tmp_path / f"readings{suffix}" + args = [str(script), "--write", str(path), "--rows", "120", "--batch-size", "37"] + if explicit: + args.insert(1, f"--{format_flag}") + if format_flag == "pytables": + args += ["--full", "station_id"] + monkeypatch.setattr(sys, "argv", args) + assert example["main"]() == 0 + + if format_flag == "parquet": + import pyarrow as pa + import pyarrow.parquet as pq + + source = pq.read_table(path) + assert source.column_names == [field.name for field in dataclasses.fields(example["Reading"])] + assert pa.types.is_large_string(source.schema.field("note").type) + assert pa.types.is_large_string(source.schema.field("message").type) + assert source.schema.field("tags").type == pa.list_(pa.int16()) + assert pa.types.is_dictionary(source.schema.field("region").type) + assert source["temperature"].null_count == 8 + assert source["humidity"].null_count == 5 + assert source["status"].null_count == 3 + assert source["note"].null_count == 3 + assert source["message"].null_count == 3 + assert source["tags"].null_count == 3 + assert source["region"].null_count == 3 + row_group = pq.read_metadata(path).row_group(0) + for name in ("id", "temperature", "note", "message"): + column = next( + row_group.column(i) + for i in range(row_group.num_columns) + if row_group.column(i).path_in_schema == name + ) + assert "RLE_DICTIONARY" not in column.encodings + + url = f"memory://{tmp_path.name}-readings{suffix}" + fsspec.filesystem("memory").pipe(url, path.read_bytes()) + if format_flag == "parquet": + with blosc2.open(url, source_format="parquet") as remote: + assert remote.to_arrow().to_pylist() == source.to_pylist() + monkeypatch.setattr(sys, "argv", [str(script), *([f"--{format_flag}"] if explicit else []), url]) + assert example["main"]() == 0 + output = capsys.readouterr().out + assert "first ids: [42]" in output + assert f"[Format: {'PyTables/HDF5' if format_flag == 'pytables' else 'Parquet'}]" in output + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_remote_example_blosc2_flag_and_inference(tmp_path, capsys, monkeypatch, explicit): + import runpy + import sys + from pathlib import Path + + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + path = tmp_path / "readings.b2z" + flag = ["--blosc2"] if explicit else [] + monkeypatch.setattr(sys, "argv", [str(script), *flag, "--write", str(path), "--rows", "120"]) + assert example["main"]() == 0 + url = f"memory://{tmp_path.name}-readings.b2z" + fsspec.filesystem("memory").pipe(url, path.read_bytes()) + monkeypatch.setattr(sys, "argv", [str(script), *flag, url]) + assert example["main"]() == 0 + assert "[Format: Blosc2 B2Z" in capsys.readouterr().out + + +def test_remote_example_unknown_extension(tmp_path, monkeypatch, capsys): + import runpy + import sys + from pathlib import Path + + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + for args in (["--write", str(tmp_path / "readings.csv")], ["memory://readings.csv"]): + monkeypatch.setattr(sys, "argv", [str(script), *args]) + with pytest.raises(SystemExit, match="2"): + example["main"]() + assert "cannot infer format" in capsys.readouterr().err + + +def test_remote_example_url_extension_ignores_query_string(monkeypatch): + import runpy + import sys + from pathlib import Path + + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + selected = [] + monkeypatch.setitem( + example["main"].__globals__, "access_external_table", lambda args: selected.append(args.parquet) + ) + monkeypatch.setattr(sys, "argv", [str(script), "https://example.com/readings.parquet?version=1"]) + assert example["main"]() == 0 + assert selected == [True] + + def test_disk_cache_metadata_key_order(tmp_path, monkeypatch): local = blosc2.CTable(dataclasses.make_dataclass("Sample", [("x", int)]), [(i,) for i in range(20)]) url = remote_table_url(tmp_path, local) From c4ebceb70ce633722e777de32a37d576ede64fa4 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 19:50:22 +0200 Subject: [PATCH 16/19] Move remote Parquet tables under RemoteStore --- doc/guides/remote_tables.md | 22 +- doc/reference/remotectable.rst | 14 +- doc/reference/remotestore.rst | 7 +- src/blosc2/b2objects.py | 4 - src/blosc2/core.py | 2 + src/blosc2/remote_ctable.py | 167 ++- src/blosc2/remote_parquet.py | 1456 +++++++++------------------ src/blosc2/remote_store.py | 311 +++++- src/blosc2/remote_store_cache.py | 9 + src/blosc2/schunk.py | 12 +- tests/ctable/test_remote_parquet.py | 91 +- tests/test_remote_store.py | 90 ++ 12 files changed, 1004 insertions(+), 1181 deletions(-) diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index eca6626bf..50332f3f3 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -5,7 +5,7 @@ Fixed-width, `blosc2.utf8()`, batch-backed variable-length, list, struct/object, and dictionary columns are fetched on demand, including their null masks. A table inside a hierarchy can also be opened through `RemoteStore`. -## Single-file Parquet (experimental) +## Single-file Parquet `blosc2.open()` recognizes `.parquet` paths, including fsspec URLs, and returns a read-only `RemoteCTable`. Use `source_format="parquet"` for an extensionless @@ -32,14 +32,28 @@ opens and cached reads need no connection to the source. Cached sources are assumed unchanged until `refresh()` is called. Use `lazy=False` to import the whole table eagerly. +Parquet is a `RemoteStore` with one root CTable. Use +`RemoteStore(url, allow_table_root=True)` and `store[""]` when a store operation +needs to own the cache and traffic counters. A Parquet file accepts only the +root selector (`""` or `"/"`); selecting a child path raises an error. +`RemoteCTable(url)` and lazy `blosc2.open(url)` use the same owner. NONE retains +no converted row groups, MEMORY shares the store budget, and DISK retains +complete converted physical-column/row-group units. A small slice can therefore +read a whole row group. Shared DISK caches support `read_cached_table()` and +offline `trim_sparse_cache()` with the same aggregate allowance. +Caches from the earlier Parquet prototype layout are not migrated; use a fresh +`cache_dir` for this RemoteStore layout. + HTTP servers must honor byte-range requests; fsspec raises a range-request error for servers that only return complete files. Download the file and import it locally when range access is unavailable. -`table.save("reference.b2z")` creates a portable archive with data already in +`table.save("reference.b2z")` creates a RemoteStore archive with data already in the cache. Opening it reuses that data and fetches missing groups on demand. -Use `include_cache=False` to save only the table metadata. Older `.b2nd` -references remain readable. +Archives of local sources keep an absolute path to the local Parquet file. +Use `include_cache=False` to save only the table metadata. `.b2z` exports use +the RemoteStore version 1 manifest with `kind="parquet"`. Runtime filesystems, credentials, +and storage options must be supplied again when reopening. Use `shared_cache=True` when separate processes need to share one `cache_dir`. They should use the same source and conversion options. diff --git a/doc/reference/remotectable.rst b/doc/reference/remotectable.rst index b756a6b55..864d44e64 100644 --- a/doc/reference/remotectable.rst +++ b/doc/reference/remotectable.rst @@ -78,13 +78,13 @@ Parquet cache details --------------------- Parquet ``cache_dir`` stores each accessed physical field and row group as a -native CTable directory beneath ``.parquet--/.b2d/``. -The generation manifest retains the table schema and row-group boundaries; a -small index in ``cache_dir`` identifies its source revision. Warm opens use -these local files without contacting the source. Cached sources are assumed -immutable until ``refresh()`` is called. The generation path itself can be -opened with :func:`blosc2.open` to recover the complete logical table and fetch -uncached groups on demand. +native CTable directory under the RemoteStore generation's ``parquet-groups`` +directory. The common manifest retains the footer, schema, conversion options, +source marker, and row-group boundaries. Warm opens reuse this discovery and +payload without contacting the source. Cached sources are assumed immutable +until ``refresh()`` is called. New portable ``.b2z`` archives use the common +RemoteStore manifest. Local Parquet sources use the same cache layout and may +export ``.b2z`` references tied to the local source path. HTTP Parquet sources need byte-range support. A persistent cache also needs a source size and version marker, such as an ETag, modification time, or diff --git a/doc/reference/remotestore.rst b/doc/reference/remotestore.rst index 904f3d11f..b5c7884f9 100644 --- a/doc/reference/remotestore.rst +++ b/doc/reference/remotestore.rst @@ -3,7 +3,7 @@ RemoteStore =========== -``RemoteStore`` discovers a read-only B2Z, Zarr or HDF5 hierarchy and returns +``RemoteStore`` discovers a read-only B2Z, Zarr, HDF5 or Parquet source and returns :ref:`RemoteArray` and :ref:`RemoteCTable` leaves. Groups and leaves share one source session: a B2Z archive, a native HDF5 index, or a Zarr store. Zarr listing remains lazy. @@ -16,6 +16,11 @@ DISK accepts ``max_cache_bytes=None`` for unbounded retention. Sources must be immutable. Generic ``blosc2.open(..., lazy=True, dataset=...)`` continues to open a single array. +A Parquet file has one CTable at its root. Pass ``allow_table_root=True`` to +retain a store handle, then use ``store[""]`` to access the table. Parquet has no +child selectors. Converted physical-column/row-group caches share the store's +policy, allowance, traffic counter, persistence, and refresh generation. + For HDF5, ``hdf5_index=`` accepts a native index dictionary, local JSON path, or remote fsspec URL. An explicit index skips hierarchy discovery and must match the source URL and selected scope. See :doc:`../guides/remote_arrays` for the diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index 539ce0307..1f1292c5d 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -128,10 +128,6 @@ def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carri if carrier is None: raise ValueError("A persisted RemoteArray requires its B2ND carrier") return blosc2.RemoteArray._from_payload(payload, carrier) - if kind == "remote_parquet": - from blosc2.remote_parquet import RemoteParquetCTable - - return RemoteParquetCTable._from_payload(payload) if kind == "lazyexpr": return decode_structured_lazyexpr(payload, carrier_path=carrier_path) if kind == "lazyudf": diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 477e3ca6f..8d686c97d 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -769,6 +769,8 @@ def parse_container_url( return urlpath, dataset, "hdf5" if any(part.endswith(".zarr") for part in parts): return urlpath, dataset, "zarr" + if parsed.path.lower().endswith(".parquet"): + return urlpath, dataset, "parquet" return urlpath, dataset, None diff --git a/src/blosc2/remote_ctable.py b/src/blosc2/remote_ctable.py index dbddb3ab9..1623e70b6 100644 --- a/src/blosc2/remote_ctable.py +++ b/src/blosc2/remote_ctable.py @@ -11,6 +11,7 @@ import operator import os +import blosc2 from blosc2.ctable import CTable from blosc2.ctable_storage import RemoteTableStorage from blosc2.remote_array import CACHE_POLICY_DEFAULT, RemoteMetadataMapping @@ -110,40 +111,13 @@ def __new__( _filesystem_resolver=None, _batch_validator=None, ): - if source_format == "parquet" or ( + parquet = source_format == "parquet" or ( isinstance(urlpath, (str, os.PathLike)) and os.fspath(urlpath).split("?", 1)[0].lower().endswith(".parquet") - ): - if source_format not in (None, "parquet"): - raise ValueError("source_format conflicts with the .parquet suffix") - from blosc2.remote_parquet import RemoteParquetCTable - - return RemoteParquetCTable( - os.fspath(urlpath), - storage_options=storage_options, - parquet_options=parquet_options, - columns=columns, - max_rows=max_rows, - string_max_length=string_max_length, - null_storage=null_storage, - auto_null_sentinels=auto_null_sentinels, - separate_nested_cols=separate_nested_cols, - list_serializer=list_serializer, - blosc2_batch_size=blosc2_batch_size, - blosc2_items_per_block=blosc2_items_per_block, - batch_size=batch_size, - cparams=cparams, - dparams=dparams, - validate=validate, - max_cache_bytes=max_cache_bytes, - cache_policy=cache_policy, - cache_dir=cache_dir, - shared_cache=shared_cache, - max_concurrency=max_concurrency, - metadata_buffer_bytes=metadata_buffer_bytes, - row_buffer_bytes=row_buffer_bytes, - ) - if ( + ) + if parquet and source_format not in (None, "parquet"): + raise ValueError("source_format conflicts with the .parquet suffix") + if not parquet and ( source_format is not None or any( value is not None @@ -180,20 +154,61 @@ def __new__( from blosc2.remote_store import RemoteStore - store = RemoteStore( - urlpath, - dataset=dataset, - path=path, - storage_options=storage_options, - cache_policy=cache_policy, - max_cache_bytes=max_cache_bytes, - cache_dir=cache_dir, - hdf5_index=hdf5_index, - _allow_array_root=True, - _filesystem=_filesystem, - _filesystem_resolver=_filesystem_resolver, - _batch_validator=_batch_validator, - ) + conversion = None + if parquet: + conversion = { + "parquet_options": parquet_options, + "columns": columns, + "max_rows": max_rows, + "string_max_length": string_max_length, + "null_storage": null_storage, + "auto_null_sentinels": auto_null_sentinels, + "separate_nested_cols": separate_nested_cols, + "list_serializer": list_serializer, + "blosc2_batch_size": blosc2_batch_size, + "blosc2_items_per_block": blosc2_items_per_block, + "batch_size": batch_size, + "cparams": cparams, + "dparams": dparams, + "validate": validate, + } + + if shared_cache: + if cache_dir is None or ( + cache_policy is not CACHE_POLICY_DEFAULT and cache_policy is not blosc2.CachePolicy.DISK + ): + raise ValueError("shared_cache=True requires a disk cache") + store = RemoteStore.with_sparse_cache( + urlpath, + cache_dir, + dataset=dataset, + path=path, + storage_options=storage_options, + max_cache_bytes=max_cache_bytes, + _filesystem=_filesystem, + _filesystem_resolver=_filesystem_resolver, + _batch_validator=_batch_validator, + _source_format="parquet" if parquet else None, + _parquet_conversion=conversion, + ) + else: + store = RemoteStore( + urlpath, + dataset=dataset, + path=path, + storage_options=storage_options, + cache_policy=cache_policy, + max_cache_bytes=max_cache_bytes, + cache_dir=cache_dir, + hdf5_index=hdf5_index, + _allow_array_root=True, + _filesystem=_filesystem, + _filesystem_resolver=_filesystem_resolver, + _batch_validator=_batch_validator, + _source_format="parquet" if parquet else None, + _parquet_conversion=conversion, + _allow_local_source=parquet, + ) try: _, full = store._resolve("") kind, diagnostic = store._owner.nodes[full] @@ -211,12 +226,16 @@ def __init__(self, *args, **kwargs): @classmethod def open_reference(cls, path, *, storage_options=None, parquet_options=None): - """Reopen a saved Parquet table with replacement runtime options.""" - from blosc2.remote_parquet import RemoteParquetCTable + """Reopen a saved Parquet RemoteStore archive.""" + if parquet_options is not None: + raise TypeError("Parquet reader options are frozen in the saved reference") + from blosc2.remote_store import RemoteStore - return RemoteParquetCTable.open_reference( - path, storage_options=storage_options, parquet_options=parquet_options - ) + table = RemoteStore._open_artifact(path, storage_options=storage_options) + if not isinstance(table, cls): + table.close() + raise ValueError("Reference does not contain a remote CTable") + return table @classmethod def with_sparse_cache( @@ -239,6 +258,7 @@ def with_sparse_cache( _source_validator=None, _manifest_validator=None, _max_nodes=None, + source_format=None, ): """Attach a remote CTable to a sparse disk cache shared across processes. @@ -247,37 +267,6 @@ def with_sparse_cache( ``max_cache_bytes=None`` for unlimited retention. For ordinary shared caching, prefer ``blosc2.open(url, cache_dir=..., shared_cache=True)``. """ - if isinstance(urlpath, (str, os.PathLike)) and os.fspath(urlpath).split("?", 1)[0].lower().endswith( - ".parquet" - ): - if any( - value is not None - for value in ( - dataset, - path, - manifest, - carrier, - _filesystem, - _filesystem_resolver, - _batch_validator, - _source_validator, - _manifest_validator, - _max_nodes, - ) - ): - raise ValueError("Parquet sparse caching accepts a single file and cache directory") - from blosc2.remote_parquet import RemoteParquetCTable - - return RemoteParquetCTable( - os.fspath(urlpath), - cache_dir=runtime_cache_path, - shared_cache=True, - max_cache_bytes=max_cache_bytes, - storage_options=storage_options, - max_concurrency=max_concurrency, - metadata_buffer_bytes=metadata_buffer_bytes, - row_buffer_bytes=row_buffer_bytes, - ) settings = { name: _positive_integer(name, value) for name, value in { @@ -304,6 +293,7 @@ def with_sparse_cache( _source_validator=_source_validator, _manifest_validator=_manifest_validator, _max_nodes=_max_nodes, + _source_format=source_format, ) try: _, full = store._resolve("") @@ -319,7 +309,14 @@ def with_sparse_cache( @classmethod def _from_owner(cls, owner, full_path, **settings): settings = {name: _positive_integer(name, value) for name, value in settings.items()} - storage = RemoteTableStorage(owner, full_path, **settings) + if owner.format == "parquet": + from blosc2.remote_parquet import ParquetTableStorage + + storage = ParquetTableStorage( + owner, owner.parquet_schema, owner.parquet_physical, owner.parquet_length, **settings + ) + else: + storage = RemoteTableStorage(owner, full_path, **settings) try: return cls._open_from_storage(storage) except BaseException: @@ -408,10 +405,12 @@ def attrs(self): @property def source(self): storage = self._remote_storage() + from blosc2.remote_store import public_source_url + return { "kind": storage._owner.format, "version": 1, - "urlpath": storage._owner.urlpath, + "urlpath": public_source_url(storage._owner.urlpath), "dataset": storage._root_key, "assume_immutable": True, } diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index e864c4da2..b028a9f3d 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -3,30 +3,24 @@ from __future__ import annotations import hashlib -import json +import math import os +import re import shutil -import tempfile -import threading import uuid import zipfile from collections import OrderedDict -from contextlib import contextmanager, nullcontext -from dataclasses import asdict, fields +from contextlib import contextmanager +from dataclasses import asdict from enum import Enum -from pathlib import Path -from urllib.parse import urlsplit, urlunsplit +from types import SimpleNamespace +from urllib.parse import urlsplit import numpy as np import blosc2 -from blosc2.core import is_fsspec_url, normalize_urlpath from blosc2.ctable import CTable, NullPolicy, get_null_policy, null_policy from blosc2.ctable_storage import RemoteTableStorage, _AllValidRows, split_field_path -from blosc2.proxy import CacheCoordinator -from blosc2.proxy_source import Traffic -from blosc2.remote_array import CACHE_POLICY_DEFAULT, normalize_cache_limit -from blosc2.remote_ctable import RemoteCTable, _positive_integer from blosc2.schema_compiler import schema_from_dict, schema_to_dict @@ -36,7 +30,7 @@ def _portable(value): if isinstance(value, Enum): value = value.value if isinstance(value, np.generic): - value = value.item() + return _portable(value.item()) if value is None or isinstance(value, bool | int | float | str | bytes): return value if isinstance(value, (list, tuple)): @@ -46,27 +40,72 @@ def _portable(value): raise TypeError(f"Nonportable Parquet reader or conversion option: {type(value).__name__}") -def _reference_options(owner): - options = { - key: value - for key, value in owner.reopen_kwargs.items() - if key - not in { - "storage_options", - "cache_dir", - "cache_policy", - "max_cache_bytes", - "shared_cache", - "_effective_null_policy", - "max_concurrency", - "metadata_buffer_bytes", - "row_buffer_bytes", - } +def parquet_identity(conversion): + import msgpack + import pyarrow + + def stable(value): + if isinstance(value, float) and not math.isfinite(value): + return str(value) + if isinstance(value, dict): + return {key: stable(item) for key, item in value.items()} + if isinstance(value, list): + return [stable(item) for item in value] + return value + + defaults = { + "parquet_options": None, + "columns": None, + "max_rows": None, + "string_max_length": None, + "null_storage": None, + "auto_null_sentinels": True, + "separate_nested_cols": True, + "list_serializer": "msgpack", + "blosc2_batch_size": 2048, + "blosc2_items_per_block": None, + "batch_size": 2048, + "cparams": None, + "dparams": None, + "validate": False, } - options["null_policy"] = { - field.name: getattr(owner.options["null_policy"], field.name) for field in fields(NullPolicy) + options = defaults | {key: value for key, value in conversion.items() if key != "_effective_null_policy"} + identity = { + "version": 1, + "blosc2": blosc2.__version__, + "pyarrow": pyarrow.__version__, + "conversion": stable(_portable(options)), + "null_policy": stable( + _portable(asdict(conversion.get("_effective_null_policy") or get_null_policy())) + ), } - return _portable(options) + return hashlib.sha256(msgpack.packb(identity, use_bin_type=True)).hexdigest() + + +def validate_parquet_metadata(metadata): + if not isinstance(metadata, dict) or not isinstance(metadata.get("parquet"), dict): + raise ValueError("Missing Parquet discovery metadata") + retained = metadata["parquet"] + discovery = retained.get("discovery") + if not isinstance(discovery, dict) or discovery.get("version") != 1: + raise ValueError("Invalid Parquet discovery metadata") + for key in ("schema", "physical", "options"): + if not isinstance(discovery.get(key), dict): + raise ValueError(f"Invalid Parquet {key}") + if not isinstance(discovery.get("arrow_metadata"), bytes): + raise ValueError("Invalid Parquet footer metadata") + if not isinstance(discovery.get("row_ends"), list) or any( + isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in discovery["row_ends"] + ): + raise ValueError("Invalid Parquet row groups") + if not isinstance(discovery.get("length"), int) or discovery["length"] < 0: + raise ValueError("Invalid Parquet length") + if not isinstance(retained.get("reader_options"), dict): + raise ValueError("Invalid Parquet reader options") + if not isinstance(retained.get("conversion"), dict) or not isinstance( + retained.get("source_marker"), dict + ): + raise ValueError("Invalid Parquet conversion or source marker") def _restore_compression_options(options): @@ -75,41 +114,12 @@ def _restore_compression_options(options): options[name] = cls(**options[name]) -def _parquet_reopen_kwargs(values): - result = { - key: values[key] - for key in ( - "storage_options", - "columns", - "max_rows", - "parquet_options", - "string_max_length", - "auto_null_sentinels", - "null_storage", - "separate_nested_cols", - "list_serializer", - "blosc2_batch_size", - "blosc2_items_per_block", - "batch_size", - "cparams", - "dparams", - "validate", - "cache_policy", - "shared_cache", - "cache_dir", - ) - } - result["max_cache_bytes"] = values["max_cache_bytes"] - result["_effective_null_policy"] = values["effective_policy"] - result.update(values["settings"]) - return result - - def _parquet_discovery(owner, schema, physical, length): import pyarrow as pa sink = pa.BufferOutputStream() owner.arrow_metadata.write_metadata_file(sink) + options = owner.parquet_options return { "version": 1, "arrow_metadata": sink.getvalue().to_pybytes(), @@ -118,8 +128,8 @@ def _parquet_discovery(owner, schema, physical, length): "row_ends": owner.row_ends.tolist(), "length": length, "options": { - **_portable({key: value for key, value in owner.options.items() if key != "null_policy"}), - "null_policy": asdict(owner.options["null_policy"]), + **_portable({key: value for key, value in options.items() if key != "null_policy"}), + "null_policy": asdict(options["null_policy"]), }, } @@ -143,12 +153,16 @@ def __getattr__(self, name): return getattr(self.handle, name) -def _source_marker(urlpath, storage_options): +def _source_marker(urlpath, storage_options, filesystem=None): import fsspec from fsspec.asyn import sync - fs, path = fsspec.core.url_to_fs(urlpath, **(storage_options or {})) - if urlsplit(urlpath).scheme in {"http", "https"}: + fs, path = ( + (filesystem, urlpath) + if filesystem is not None + else fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + ) + if urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session"): try: info = sync(fs.loop, _http_source_info, fs, path) except (OSError, RuntimeError): @@ -196,123 +210,29 @@ async def _http_source_info(fs, path): return info -def _open_source_handle(urlpath, storage_options, marker, traffic): +def _open_source_handle(urlpath, storage_options, marker, traffic, filesystem=None): import fsspec - fs, path = fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + fs, path = ( + (filesystem, urlpath) + if filesystem is not None + else fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + ) kwargs = ( - {"size": int(marker["size"])} if marker and urlsplit(urlpath).scheme in {"http", "https"} else {} + {"size": int(marker["size"])} + if marker and urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session") + else {} ) return _CountingHandle(fs.open(path, "rb", **kwargs), traffic) -def _source_url(urlpath): - urlpath = str(normalize_urlpath(urlpath)) - if not is_fsspec_url(urlpath): - return os.path.abspath(urlpath) - url = urlsplit(urlpath) - return urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, url.query, "")) - - -def _cache_marker_index(urlpath, storage_options, options, cache_dir): - import pyarrow - - if not is_fsspec_url(urlpath): - urlpath = os.path.abspath(urlpath) - identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, options) - digest = hashlib.sha256(repr(identity).encode()).hexdigest() - return Path(cache_dir) / f".parquet-marker-{digest}.json" - - -def _cached_source_marker(urlpath, storage_options, options, cache_dir): - path = _cache_marker_index(urlpath, storage_options, options, cache_dir) - try: - value = json.loads(path.read_text()) - marker, directory = value["marker"], value["directory"] - if ( - value["version"] == 1 - and isinstance(marker, dict) - and "size" in marker - and len(marker) > 1 - and isinstance(directory, str) - and directory == Path(directory).name - and (path.parent / directory / "active_generation.json").is_file() - ): - return marker - except (OSError, ValueError, KeyError, TypeError): - pass - return None - - -def _publish_source_marker(urlpath, storage_options, options, cache_dir, marker, disk): - from blosc2.remote_store_cache import atomic_write - - path = _cache_marker_index(urlpath, storage_options, options, cache_dir) - atomic_write(path, json.dumps({"version": 1, "marker": marker, "directory": disk.path.name}).encode()) - - -def _disk_cache_path(urlpath, storage_options, options, cache_dir, marker, *, shared=False): - import pyarrow - - from blosc2.remote_store_cache import SharedStoreCache, StoreDiskCache - - if "size" not in marker or len(marker) < 2: - raise ValueError("A persistent Parquet cache requires source size and a version marker") - if not is_fsspec_url(urlpath): - urlpath = os.path.abspath(urlpath) - identity = (1, blosc2.__version__, pyarrow.__version__, urlpath, storage_options, marker, options) - digest = hashlib.sha256(repr(identity).encode()).hexdigest() - source = { - "kind": "parquet", - "urlpath": _source_url(urlpath), - "identity": digest, - } - if is_fsspec_url(urlpath): - from blosc2.remote_array import validate_persistable_url - - validate_persistable_url(source["urlpath"]) - disk = (SharedStoreCache if shared else StoreDiskCache)(cache_dir, source) - try: - with _disk_guard(disk): - manifest = disk.load() - if manifest is None: - manifest = { - "version": 1, - "source": source, - "generation": uuid.uuid4().hex, - "nodes": {"": ("ctable", {})}, - "attrs": {}, - "listed": {}, - "metadata": {"source_marker": marker}, - "caches": [], - } - disk.publish(manifest) - return disk, disk.path / f"{manifest['generation']}.b2d" - except BaseException: - disk.close() - raise - - -def _disk_guard(disk): - return disk.guard() if disk is not None and hasattr(disk, "guard") else nullcontext() - - def _group_filename(number, physical): token = hashlib.sha256(physical.encode()).hexdigest()[:16] return f"{number}-{token}.b2d" -def open_parquet_cache_artifact(path, *, mode="r", **kwargs): - if os.path.isdir(path): - carrier = blosc2.blosc2_ext.open(str(Path(path) / "embed.b2e"), "r", 0) - else: - with zipfile.ZipFile(path) as archive: - carrier = blosc2.schunk_from_cframe(archive.read("embed.b2e")) - if carrier.vlmeta["b2remote_manifest"]["source"].get("kind") != "parquet": - return None - if os.path.isdir(path): - return RemoteParquetCTable._open_cache_artifact(path, mode=mode, **kwargs) - return RemoteParquetCTable._open_archive_artifact(path, mode=mode, **kwargs) +def valid_group_name(name): + return isinstance(name, str) and re.fullmatch(r"[0-9]+-[0-9a-f]{16}\.b2d", name) is not None def _projected_field(name, physical, paths): @@ -329,106 +249,246 @@ def _projected_field(name, physical, paths): return matches.pop() if len(matches) == 1 else physical -class _ParquetOwner: - format = "parquet" - is_mutable = True - shared = False +def discover_parquet(owner, conversion): # noqa: C901 + """Build the one CTable root and keep only the Arrow reader on the store owner.""" + import pyarrow as pa + import pyarrow.parquet as pq - def __init__( - self, - path, - handle, - parquet_file, - options, - *, - cache_policy, - max_cache_bytes, - cache_dir, - disk, - storage_options=None, - source_marker=None, - arrow_metadata=None, - parquet_options=None, - ): - self.urlpath = path - self.handle = handle - self.traffic = handle.traffic if handle is not None else Traffic() - self.storage_options = storage_options - self.source_marker = source_marker - self.arrow_metadata = arrow_metadata or parquet_file.metadata - self.parquet_options = parquet_options - self.parquet_file = parquet_file - self.options = options - self.lock = threading.RLock() - self.generation = 0 - self.users = 0 - self.closed = False - self.cache = OrderedDict() - self.cache_bytes = 0 - self.cache_policy = cache_policy - self.max_cache_bytes = max_cache_bytes - self.cache_dir = cache_dir - self.disk = disk + if owner.root: + raise ValueError("Parquet files support only the root table") + retained = owner.metadata.get("parquet") + if retained is not None: + try: + discovery = retained["discovery"] + schema = schema_from_dict(discovery["schema"]) + physical = discovery["physical"] + ends = np.asarray(discovery["row_ends"], dtype=np.int64) + metadata = pq.read_metadata(pa.BufferReader(discovery["arrow_metadata"])) + if ( + discovery["version"] != 1 + or set(physical) != {column.name for column in schema.columns} + or ends.ndim != 1 + or len(ends) > metadata.num_row_groups + or np.any(ends < 0) + or np.any(ends[1:] < ends[:-1]) + or not isinstance(discovery["length"], int) + or discovery["length"] < 0 + ): + raise ValueError("Invalid retained Parquet metadata") + options = dict(discovery["options"]) + options["null_policy"] = NullPolicy(**options["null_policy"]) + _restore_compression_options(options) + except (KeyError, TypeError, ValueError, OSError) as exc: + raise ValueError("Invalid retained Parquet metadata") from exc + owner.arrow_metadata = metadata + validate_parquet_groups(owner, metadata) + owner.row_ends = ends + owner.parquet_schema = schema + owner.parquet_physical = physical + owner.parquet_length = discovery["length"] + owner.parquet_options = options + owner.parquet_reader_options = retained.get("reader_options") or {} + owner.parquet_source_marker = retained.get("source_marker") + owner.nodes[""] = ("ctable", {"kind": "ctable"}) + return + + reader_options = conversion.get("parquet_options") or {} + if reader_options.get("memory_map"): + raise ValueError("memory_map is incompatible with a remote Parquet handle") + columns = conversion.get("columns") + if columns is not None and len(set(columns)) != len(columns): + raise ValueError("columns must be unique") + max_rows = conversion.get("max_rows") + if max_rows is not None and max_rows < 0: + raise ValueError("max_rows must be non-negative") + batch_size = conversion.get("batch_size", 2048) + CTable._validate_arrow_batch_size(batch_size) + policy = conversion.get("_effective_null_policy") or get_null_policy() + options = { + "null_policy": policy, + "string_max_length": conversion.get("string_max_length"), + "auto_null_sentinels": conversion.get("auto_null_sentinels", True), + "null_storage": conversion.get("null_storage"), + "separate_nested_cols": conversion.get("separate_nested_cols", True), + "list_serializer": conversion.get("list_serializer", "msgpack"), + "blosc2_batch_size": conversion.get("blosc2_batch_size", 2048), + "blosc2_items_per_block": conversion.get("blosc2_items_per_block"), + "batch_size": batch_size, + "cparams": conversion.get("cparams"), + "dparams": conversion.get("dparams"), + "validate": conversion.get("validate", False), + } + marker = ( + _source_marker(owner.urlpath, owner.storage_options, owner.filesystem) + if owner.parquet_persistent + else None + ) + if marker is not None and ("size" not in marker or len(marker) < 2): + raise ValueError("A persistent Parquet cache requires source size and a version marker") + handle = _open_source_handle( + owner.urlpath, owner.storage_options, marker, owner.traffic, owner.filesystem + ) + pf = None + try: + pf = pq.ParquetFile(handle, **reader_options) + fields = pf.schema_arrow + if columns is not None: + fields = pa.schema([fields.field(name) for name in columns]) + root_name = "root" + if "" in fields.names: + while root_name in fields.names: + root_name += "_1" + flatten_root = options["separate_nested_cols"] and CTable._detect_unnamed_root_list_struct( + pa, fields + ) + sample_needed = (options["null_storage"] or policy.resolve_null_storage()) != "mask" + sample = ( + next(pf.iter_batches(batch_size=1 if flatten_root else batch_size, columns=columns), None) + if sample_needed + else None + ) + if sample is None: + sample = pa.RecordBatch.from_arrays([pa.array([], type=f.type) for f in fields], schema=fields) + if not flatten_root and "" in fields.names: + sample = sample.rename_columns([root_name if name == "" else name for name in fields.names]) + meta = dict(sample.schema.metadata or {}) + meta[b"blosc2_empty_root_physical"] = root_name.encode() + sample = sample.replace_schema_metadata(meta) + options.update(root_name=root_name, flatten_root=flatten_root) + with null_policy(policy): + probe = CTable.from_arrow( + sample.schema, + [sample], + **{ + key: options[key] + for key in ( + "string_max_length", + "auto_null_sentinels", + "null_storage", + "separate_nested_cols", + "list_serializer", + "blosc2_batch_size", + "blosc2_items_per_block", + "cparams", + "dparams", + "validate", + ) + }, + ) + physical = { + name: ( + "" + if flatten_root or (split_field_path(name)[0] == root_name and "" in fields.names) + else name + if name in fields.names + else split_field_path(name)[0] + ) + for name in probe.col_names + } + paths = [pf.schema.column(i).path for i in range(len(pf.schema.names))] + physical = { + name: _projected_field(name, source, paths) + if flatten_root or (source and source != name) + else source + for name, source in physical.items() + } + if flatten_root: + lengths = [] + count = 0 + for group in range(pf.num_row_groups): + if max_rows is not None and count >= max_rows: + break + row_group = pf.metadata.row_group(group) + leaf = min( + (row_group.column(i) for i in range(row_group.num_columns)), + key=lambda column: column.total_compressed_size, + ) + arr = pf.read_row_group(group, columns=[leaf.path_in_schema]).column(0) + size = pa.compute.sum(pa.compute.list_value_length(arr)).as_py() or 0 + lengths.append(size) + count += size + ends = np.cumsum(lengths) + else: + ends = np.cumsum([pf.metadata.row_group(i).num_rows for i in range(pf.num_row_groups)]) + length = int(ends[-1]) if len(ends) else 0 + if max_rows is not None: + length = min(length, max_rows) + owner.parquet_handle = handle + owner.parquet_file = pf + owner.arrow_metadata = pf.metadata + validate_parquet_groups(owner, pf.metadata) + owner.parquet_schema = probe._schema + owner.parquet_physical = physical + owner.parquet_length = length + owner.parquet_options = options + owner.parquet_reader_options = reader_options + owner.parquet_source_marker = marker + owner.row_ends = ends + discovery = _parquet_discovery(owner, probe._schema, physical, length) + owner.metadata["parquet"] = { + "discovery": discovery, + "reader_options": _portable(reader_options), + "source_marker": marker, + "conversion": _portable(conversion), + } + owner.nodes[""] = ("ctable", {"kind": "ctable"}) + probe.close() + except BaseException: + if pf is not None: + pf.close() + handle.close() + raise + + +def validate_parquet_groups(owner, footer): + if owner.source_validator is None: + return + for number in range(footer.num_row_groups): + size = max(1, footer.row_group(number).total_byte_size) + geometry = SimpleNamespace(shape=(size,), dtype=np.dtype("u1"), chunks=(size,), blocks=(size,)) + owner.source_validator(geometry) + + +class ParquetCache: + """One cache leaf for converted Parquet row groups in a RemoteStore.""" + + def __init__(self, owner): + self.owner = owner self._cache_key = "parquet" self._cache_sizes = {} self._cache_lru = OrderedDict() - self.cache_coordinator = CacheCoordinator(max_cache_bytes) if disk is not None else None - if self.cache_coordinator is not None: - with _disk_guard(self.disk): - self._sync_evictions() - self.cache_coordinator.register(self) - self._cache_manager = None - self.row_ends = np.cumsum( - [self.arrow_metadata.row_group(i).num_rows for i in range(self.arrow_metadata.num_row_groups)] + self.cache = OrderedDict() + self.cache_bytes = 0 + self.cache_dir = ( + owner.disk.path / f"{owner.generation}.b2d" / "parquet-groups" + if owner.disk is not None + else getattr(owner, "parquet_artifact_dir", None) ) + if self.cache_dir is not None: + self.cache_dir.mkdir(parents=True, exist_ok=True) + owner.cache_coordinator.register(self) + self._sync_evictions() - def acquire(self): - self.users += 1 - - def release(self): - self.users -= 1 - if self.users == 0: - self.closed = True - self.cache.clear() - if self.parquet_file is not None: - self.parquet_file.close() - if self.handle is not None: - self.handle.close() - if self.disk is not None: - self.disk.close() - if self._cache_manager is not None: - self._cache_manager.cleanup() - - @contextmanager - def group(self, number, physical): - with self.lock, _disk_guard(self.disk): - table = self._group_locked(number, physical, (number, physical)) - try: - yield table - finally: - if self.cache_policy is not blosc2.CachePolicy.MEMORY: - table.close() + _filename = staticmethod(_group_filename) def _sync_evictions(self): if self.cache_dir is None: return - # ponytail: scan cached groups on each accounting pass; persist sizes if directory walks dominate reads. entries = sorted(self.cache_dir.glob("*-*.b2d"), key=lambda path: path.stat().st_mtime_ns) present = {path.name for path in entries} - for name in tuple(self._cache_sizes): + for name in (key for key in tuple(self._cache_sizes) if isinstance(key, str)): if name not in present: self._cache_sizes.pop(name) self._cache_lru.pop(name, None) - if self.cache_coordinator is not None: - self.cache_coordinator.forget(self, name) + self.owner.cache_coordinator.forget(self, name) for path in entries: if path.name not in self._cache_sizes: self._cache_sizes[path.name] = sum( file.stat().st_size for file in path.rglob("*") if file.is_file() ) self._cache_lru[path.name] = None - if self.cache_coordinator is not None: - self.cache_coordinator.touch(self, path.name) + if self.owner.is_mutable: + self.owner.cache_coordinator.touch(self, path.name) def _retained_cache_bytes(self): return sum(self._cache_sizes.values()) @@ -436,7 +496,9 @@ def _retained_cache_bytes(self): def _trim_cache(self, target_bytes, *, max_chunks=None): evicted = [] while ( - self._retained_cache_bytes() > target_bytes + self.cache_dir is not None + and self.owner.is_mutable + and self._retained_cache_bytes() > target_bytes and self._cache_lru and (max_chunks is None or len(evicted) < max_chunks) ): @@ -444,95 +506,148 @@ def _trim_cache(self, target_bytes, *, max_chunks=None): shutil.rmtree(self.cache_dir / name) self._cache_lru.pop(name) self._cache_sizes.pop(name) - self.cache_coordinator.forget(self, name) + self.owner.cache_coordinator.forget(self, name) evicted.append(name) + while ( + self._retained_cache_bytes() > target_bytes + and self.cache + and (max_chunks is None or len(evicted) < max_chunks) + ): + key, (table, size) = self.cache.popitem(last=False) + self.cache_bytes -= size + self._cache_sizes.pop(key, None) + self._cache_lru.pop(key, None) + self.owner.cache_coordinator.forget(self, key) + table.close() + evicted.append(key) return tuple(evicted) - def _touch_group(self, path): - if self.cache_coordinator is None: - return - path.touch() - self._cache_lru.pop(path.name, None) - self._cache_lru[path.name] = None - self.cache_coordinator.touch(self, path.name) - - def _group_locked(self, number, physical, key): - if key in self.cache: - self.cache.move_to_end(key) - return self.cache[key][0] - cache_file = None - if self.cache_dir is not None: - cache_file = self.cache_dir / _group_filename(number, physical) - if cache_file.exists(): + @contextmanager + def group(self, number, physical): # noqa: C901 + owner = self.owner + with owner.lock: + from blosc2.remote_store import CacheMiss + + key = (number, physical) + if key in self.cache: + self.cache.move_to_end(key) + owner.cache_coordinator.touch(self, key) + yield self.cache[key][0] + return + path = None if self.cache_dir is None else self.cache_dir / _group_filename(number, physical) + cache_file = path if path is not None and (owner.disk is not None or path.exists()) else None + if cache_file is not None and cache_file.exists(): try: - cached = CTable.open(cache_file) - start = 0 if number == 0 else int(self.row_ends[number - 1]) - if len(cached) == int(self.row_ends[number]) - start: - self._touch_group(cache_file) - return cached - cached.close() + table = CTable.open(cache_file) + start = 0 if number == 0 else int(owner.row_ends[number - 1]) + if len(table) == int(owner.row_ends[number]) - start: + cache_file.touch() + if owner.is_mutable: + owner.cache_coordinator.touch(self, cache_file.name) + try: + yield table + finally: + table.close() + return + table.close() except (OSError, ValueError, RuntimeError, KeyError, TypeError, zipfile.BadZipFile): - pass # Interrupted or corrupt publication: rebuild this group. - if self.parquet_file is None: - self.handle = _open_source_handle( - self.urlpath, self.storage_options, self.source_marker, self.traffic - ) - try: + pass + if owner.cache_coordinator.cached_only: + raise CacheMiss + if owner.parquet_file is None: import pyarrow.parquet as pq - self.parquet_file = pq.ParquetFile( - self.handle, metadata=self.arrow_metadata, **(self.parquet_options or {}) + if owner.artifact_path is not None and owner.parquet_source_marker is not None: + marker = _source_marker(owner.urlpath, owner.storage_options, owner.filesystem) + if marker != owner.parquet_source_marker: + raise RuntimeError("Parquet reference source has changed") + owner.parquet_handle = _open_source_handle( + owner.urlpath, + owner.storage_options, + owner.parquet_source_marker, + owner.traffic, + owner.filesystem, ) - except BaseException: - self.handle.close() - self.handle = None - raise - arrow = self.parquet_file.read_row_group(number, columns=[physical], use_threads=False) - if physical == "" and not self.options["flatten_root"]: - arrow = arrow.rename_columns([self.options["root_name"]]) - meta = dict(arrow.schema.metadata or {}) - meta[b"blosc2_empty_root_physical"] = self.options["root_name"].encode() - arrow = arrow.replace_schema_metadata(meta) - with null_policy(self.options["null_policy"]): - table = CTable.from_arrow( - arrow.schema, - arrow.to_batches(max_chunksize=self.options["batch_size"]), - string_max_length=self.options["string_max_length"], - auto_null_sentinels=self.options["auto_null_sentinels"], - null_storage=self.options["null_storage"], - separate_nested_cols=self.options["separate_nested_cols"], - list_serializer=self.options["list_serializer"], - blosc2_batch_size=self.options["blosc2_batch_size"], - blosc2_items_per_block=self.options["blosc2_items_per_block"], - cparams=self.options["cparams"], - dparams=self.options["dparams"], - validate=self.options["validate"], - capacity_hint=int(self.row_ends[number]) - (int(self.row_ends[number - 1]) if number else 0), - ) - size = int(getattr(table, "cbytes", 0) or arrow.nbytes) - if cache_file is not None: - temporary = cache_file.with_name(f".{uuid.uuid4().hex}.b2d") + try: + owner.parquet_file = pq.ParquetFile( + owner.parquet_handle, + metadata=owner.arrow_metadata, + **owner.parquet_reader_options, + ) + except BaseException: + owner.parquet_handle.close() + owner.parquet_handle = None + raise + arrow = owner.parquet_file.read_row_group(number, columns=[physical], use_threads=False) + options = owner.parquet_options + if physical == "" and not options["flatten_root"]: + arrow = arrow.rename_columns([options["root_name"]]) + meta = dict(arrow.schema.metadata or {}) + meta[b"blosc2_empty_root_physical"] = options["root_name"].encode() + arrow = arrow.replace_schema_metadata(meta) + with null_policy(options["null_policy"]): + table = CTable.from_arrow( + arrow.schema, + arrow.to_batches(max_chunksize=options["batch_size"]), + **{ + name: options[name] + for name in ( + "string_max_length", + "auto_null_sentinels", + "null_storage", + "separate_nested_cols", + "list_serializer", + "blosc2_batch_size", + "blosc2_items_per_block", + "cparams", + "dparams", + "validate", + ) + }, + capacity_hint=int(owner.row_ends[number]) + - (int(owner.row_ends[number - 1]) if number else 0), + ) + size = int(getattr(table, "cbytes", 0) or arrow.nbytes) + if cache_file is not None and owner.disk is not None: + temporary = cache_file.with_name(f".{uuid.uuid4().hex}.b2d") + try: + with table.copy(urlpath=temporary): + pass + if cache_file.exists(): + shutil.rmtree(cache_file) + os.replace(temporary, cache_file) + finally: + if temporary.exists(): + shutil.rmtree(temporary) + self._sync_evictions() + owner.cache_coordinator.touch(self, cache_file.name) + owner.cache_coordinator.enforce() + owner.save_manifest() + elif owner.cache_policy is blosc2.CachePolicy.MEMORY or ( + owner.disk is None and owner.cache_policy is not blosc2.CachePolicy.NONE + ): + pass try: - with table.copy(urlpath=temporary): - pass - if cache_file.exists(): - shutil.rmtree(cache_file) - os.replace(temporary, cache_file) + yield table finally: - if temporary.exists(): - shutil.rmtree(temporary) - self._sync_evictions() - self._touch_group(cache_file) - self.cache_coordinator.enforce() - elif self.cache_policy is blosc2.CachePolicy.MEMORY: - self.cache[key] = table, size - self.cache_bytes += size - while ( - self.max_cache_bytes is not None and self.cache_bytes > self.max_cache_bytes and self.cache - ): - _, (_, removed) = self.cache.popitem(last=False) - self.cache_bytes -= removed - return table + if owner.cache_policy is blosc2.CachePolicy.MEMORY or ( + owner.disk is None and owner.cache_policy is not blosc2.CachePolicy.NONE + ): + self.cache[key] = table, size + self.cache_bytes += size + self._cache_sizes[key] = size + self._cache_lru[key] = None + owner.cache_coordinator.touch(self, key) + owner.cache_coordinator.enforce() + else: + table.close() + + def close(self): + for table, _ in self.cache.values(): + table.close() + self.cache.clear() + self._cache_sizes.clear() + self._cache_lru.clear() class _ParquetColumn: @@ -637,7 +752,15 @@ def _getitem_locked(self, key): class ParquetTableStorage(RemoteTableStorage): def __init__( - self, owner, schema, physical, length, *, max_concurrency, metadata_buffer_bytes, row_buffer_bytes + self, + owner, + schema, + physical, + length, + *, + max_concurrency=8, + metadata_buffer_bytes=8 << 20, + row_buffer_bytes=64 << 20, ): self._owner = owner self._generation = owner.generation @@ -694,672 +817,3 @@ def close(self): if not self._closed: self._closed = True self._owner.release() - - -class RemoteParquetCTable(RemoteCTable): - """A read-only Parquet CTable that converts accessed row groups on demand.""" - - def __new__( # noqa: C901 - cls, - urlpath, - *, - storage_options=None, - columns=None, - max_rows=None, - parquet_options=None, - string_max_length=None, - auto_null_sentinels=True, - null_storage=None, - separate_nested_cols=True, - list_serializer="msgpack", - blosc2_batch_size=2048, - blosc2_items_per_block=None, - batch_size=2048, - cparams=None, - dparams=None, - validate=False, - max_cache_bytes=CACHE_POLICY_DEFAULT, - cache_policy=CACHE_POLICY_DEFAULT, - shared_cache=False, - max_concurrency=8, - metadata_buffer_bytes=8 << 20, - row_buffer_bytes=64 << 20, - _effective_null_policy=None, - **kwargs, - ): - import pyarrow as pa - import pyarrow.parquet as pq - - urlpath = str(normalize_urlpath(urlpath)) - cache_dir = kwargs.pop("cache_dir", None) - seed_cache = kwargs.pop("_seed_cache", None) - cache_manager = kwargs.pop("_cache_manager", None) - refresh_marker = kwargs.pop("_refresh_marker", None) - if kwargs: - raise TypeError(f"Unsupported Parquet options: {', '.join(kwargs)}") - if max_rows is not None and max_rows < 0: - raise ValueError("max_rows must be non-negative") - CTable._validate_arrow_batch_size(batch_size) - settings = { - name: _positive_integer(name, value) - for name, value in { - "max_concurrency": max_concurrency, - "metadata_buffer_bytes": metadata_buffer_bytes, - "row_buffer_bytes": row_buffer_bytes, - }.items() - } - if columns is not None and len(set(columns)) != len(columns): - raise ValueError("columns must be unique") - if parquet_options and parquet_options.get("memory_map"): - raise ValueError("memory_map is incompatible with a remote Parquet handle") - if cache_policy is CACHE_POLICY_DEFAULT: - cache_policy = blosc2.CachePolicy.DISK if cache_dir is not None else blosc2.CachePolicy.MEMORY - else: - cache_policy = blosc2.CachePolicy(cache_policy) - if (cache_policy is blosc2.CachePolicy.DISK) != (cache_dir is not None): - raise ValueError("Parquet DISK cache policy requires cache_dir, and cache_dir requires DISK") - if shared_cache and cache_policy is not blosc2.CachePolicy.DISK: - raise ValueError("shared_cache=True requires a disk cache") - max_cache_bytes = normalize_cache_limit(cache_policy, max_cache_bytes) - effective_policy = _effective_null_policy or get_null_policy() - options = { - "null_policy": effective_policy, - "string_max_length": string_max_length, - "auto_null_sentinels": auto_null_sentinels, - "null_storage": null_storage, - "separate_nested_cols": separate_nested_cols, - "list_serializer": list_serializer, - "blosc2_batch_size": blosc2_batch_size, - "blosc2_items_per_block": blosc2_items_per_block, - "batch_size": batch_size, - "cparams": cparams, - "dparams": dparams, - "validate": validate, - } - identity_options = (columns, max_rows, parquet_options, _portable(options)) - cached_marker = ( - _cached_source_marker(urlpath, storage_options, identity_options, cache_dir) - if cache_dir is not None and refresh_marker is None - else None - ) - marker = ( - refresh_marker or cached_marker or _source_marker(urlpath, storage_options) - if cache_dir is not None - else None - ) - disk, disk_dir = ( - _disk_cache_path( - urlpath, storage_options, identity_options, cache_dir, marker, shared=shared_cache - ) - if cache_dir is not None - else (None, None) - ) - handle = None - try: - with _disk_guard(disk): - discovery = disk.load()["metadata"].get("parquet_discovery") if disk is not None else None - if discovery is not None: - try: - schema = schema_from_dict(discovery["schema"]) - physical = discovery["physical"] - ends = np.asarray(discovery["row_ends"], dtype=np.int64) - length = discovery["length"] - metadata = pq.read_metadata(pa.BufferReader(discovery["arrow_metadata"])) - if ( - discovery["version"] != 1 - or not isinstance(physical, dict) - or set(physical) != {column.name for column in schema.columns} - or not isinstance(length, int) - or length < 0 - or ends.ndim != 1 - or len(ends) > metadata.num_row_groups - or np.any(ends < 0) - or np.any(ends[1:] < ends[:-1]) - or (len(ends) and length > ends[-1]) - ): - raise ValueError("Invalid retained Parquet metadata") - restored_options = dict(discovery["options"]) - restored_options["null_policy"] = NullPolicy(**restored_options["null_policy"]) - _restore_compression_options(restored_options) - except (KeyError, TypeError, ValueError, OSError): - discovery = None - else: - owner = _ParquetOwner( - urlpath, - None, - None, - restored_options, - cache_policy=cache_policy, - max_cache_bytes=max_cache_bytes, - cache_dir=disk_dir, - disk=disk, - storage_options=storage_options, - source_marker=marker, - arrow_metadata=metadata, - parquet_options=parquet_options, - ) - owner.row_ends = ends - owner.identity_options = identity_options - owner.discovery = discovery - owner._cache_manager = cache_manager - owner.reopen_kwargs = _parquet_reopen_kwargs(locals()) - storage = ParquetTableStorage(owner, schema, physical, length, **settings) - try: - table = cls._open_from_storage(storage) - if cached_marker is None: - _publish_source_marker( - urlpath, storage_options, identity_options, cache_dir, marker, disk - ) - return table - except BaseException: - storage.close() - raise - handle = _open_source_handle(urlpath, storage_options, marker, Traffic()) - pf = pq.ParquetFile(handle, **(parquet_options or {})) - fields = pf.schema_arrow - if columns is not None: - fields = pa.schema([fields.field(name) for name in columns]) - root_name = "root" - if "" in fields.names: - while root_name in fields.names: - root_name += "_1" - flatten_root = separate_nested_cols and CTable._detect_unnamed_root_list_struct(pa, fields) - # Mask-backed schemas are determined by the Arrow schema alone. A - # value sample is only needed when in-band null handling may reject - # actual nulls during inference. - sample_needed = (null_storage or effective_policy.resolve_null_storage()) != "mask" - sample = ( - next(pf.iter_batches(batch_size=1 if flatten_root else batch_size, columns=columns), None) - if sample_needed - else None - ) - if sample is None: - sample = pa.RecordBatch.from_arrays( - [pa.array([], type=f.type) for f in fields], schema=fields - ) - if not flatten_root and "" in fields.names: - names = [root_name if name == "" else name for name in fields.names] - sample = sample.rename_columns(names) - meta = dict(sample.schema.metadata or {}) - meta[b"blosc2_empty_root_physical"] = root_name.encode() - sample = sample.replace_schema_metadata(meta) - options["root_name"] = root_name - options["flatten_root"] = flatten_root - if seed_cache: - if disk_dir is None: - raise ValueError("A retained Parquet cache requires cache_dir") - with _disk_guard(disk): - for name, data in seed_cache.items(): - if ( - not isinstance(name, str) - or not isinstance(data, bytes) - or (not name.startswith("row-map-") and not name.endswith(".b2z")) - or Path(name).name != name - ): - raise ValueError("Invalid retained Parquet cache entry") - destination = disk_dir / (name[:-4] + ".b2d" if name.endswith(".b2z") else name) - with tempfile.NamedTemporaryFile( - dir=disk_dir, suffix=Path(name).suffix, delete=False - ) as staged: - staged.write(data) - try: - if name.endswith(".b2z"): - with CTable.load(staged.name) as table: - with table.copy(urlpath=destination): - pass - else: - os.replace(staged.name, destination) - finally: - Path(staged.name).unlink(missing_ok=True) - with null_policy(options["null_policy"]): - probe = CTable.from_arrow( - sample.schema, - [sample], - string_max_length=string_max_length, - auto_null_sentinels=auto_null_sentinels, - null_storage=null_storage, - separate_nested_cols=separate_nested_cols, - list_serializer=list_serializer, - blosc2_batch_size=blosc2_batch_size, - blosc2_items_per_block=blosc2_items_per_block, - cparams=cparams, - dparams=dparams, - validate=validate, - ) - physical = { - name: ( - "" - if flatten_root or (split_field_path(name)[0] == root_name and "" in fields.names) - else name - if name in fields.names - else split_field_path(name)[0] - ) - for name in probe.col_names - } - paths = [pf.schema.column(i).path for i in range(len(pf.schema.names))] - physical = { - name: _projected_field(name, source, paths) - if flatten_root or (source and source != name) - else source - for name, source in physical.items() - } - if flatten_root: - # ponytail: full length scan; replace with a persisted offsets map when large nested roots matter. - with _disk_guard(disk): - lengths = [] - count = 0 - map_file = disk_dir / f"row-map-{max_rows}.npy" if disk_dir is not None else None - ends = None - if map_file is not None and map_file.exists(): - try: - if map_file.stat().st_size <= 1024 + 8 * pf.num_row_groups: - loaded = np.load(map_file, allow_pickle=False) - if ( - loaded.ndim == 1 - and loaded.dtype == np.dtype(np.int64) - and len(loaded) <= pf.num_row_groups - and np.all(loaded >= 0) - and np.all(loaded[1:] >= loaded[:-1]) - and ( - len(loaded) == pf.num_row_groups - or max_rows == 0 - or (max_rows is not None and len(loaded) and loaded[-1] >= max_rows) - ) - ): - ends = loaded - except (OSError, ValueError): - pass - if ends is None: - for group in range(pf.num_row_groups): - if max_rows is not None and count >= max_rows: - break - row_group = pf.metadata.row_group(group) - leaf = min( - (row_group.column(i) for i in range(row_group.num_columns)), - key=lambda column: column.total_compressed_size, - ) - arr = pf.read_row_group(group, columns=[leaf.path_in_schema]).column(0) - size = pa.compute.sum(pa.compute.list_value_length(arr)).as_py() or 0 - lengths.append(size) - count += size - ends = np.cumsum(lengths) - if map_file is not None: - temporary = map_file.with_name(f".{uuid.uuid4().hex}.npy") - try: - np.save(temporary, ends) - os.replace(temporary, map_file) - finally: - temporary.unlink(missing_ok=True) - else: - ends = np.cumsum([pf.metadata.row_group(i).num_rows for i in range(pf.num_row_groups)]) - length = int(ends[-1]) if len(ends) else 0 - if max_rows is not None: - length = min(length, max_rows) - owner = _ParquetOwner( - urlpath, - handle, - pf, - options, - cache_policy=cache_policy, - max_cache_bytes=max_cache_bytes, - cache_dir=disk_dir, - disk=disk, - storage_options=storage_options, - source_marker=marker, - parquet_options=parquet_options, - ) - owner.source_marker = marker - owner.identity_options = identity_options - owner._cache_manager = cache_manager - owner.reopen_kwargs = _parquet_reopen_kwargs(locals()) - owner.row_ends = ends - owner.discovery = _parquet_discovery(owner, probe._schema, physical, length) - if disk is not None: - with _disk_guard(disk): - manifest = disk.load() - try: - manifest["metadata"]["reopen"] = _reference_options(owner) - except TypeError: - manifest["metadata"]["reopen"] = None - manifest["metadata"]["parquet_discovery"] = owner.discovery - disk.publish(manifest) - if disk is not None and cached_marker is None: - _publish_source_marker(urlpath, storage_options, identity_options, cache_dir, marker, disk) - storage = ParquetTableStorage(owner, probe._schema, physical, length, **settings) - probe.close() - try: - return cls._open_from_storage(storage) - except BaseException: - storage.close() - raise - except BaseException: - if handle is not None: - handle.close() - if disk is not None: - disk.close() - raise - - def refresh(self): - storage = self._remote_storage() - owner = storage._owner - with owner.lock: - refresh_marker = ( - _source_marker(owner.urlpath, owner.storage_options) if owner.disk is not None else None - ) - if refresh_marker is not None and refresh_marker == owner.source_marker: - return - fresh = type(self)(owner.urlpath, _refresh_marker=refresh_marker, **owner.reopen_kwargs) - fresh._remote_storage()._owner._cache_manager = owner._cache_manager - owner._cache_manager = None - owner.generation += 1 - state = fresh.__dict__.copy() - fresh._storage = None - self.__dict__ = state - self._cols._table = self - storage.close() - - @property - def source(self): - source_url = _source_url(self._remote_storage()._owner.urlpath) - if is_fsspec_url(source_url): - from blosc2.remote_array import validate_persistable_url - - try: - validate_persistable_url(source_url) - except ValueError: - source_url = source_url.split("?", 1)[0] - return { - "kind": "parquet", - "version": 1, - "urlpath": source_url, - } - - @property - def traffic(self): - return self._remote_storage()._owner.traffic - - @property - def cache_bytes(self): - owner = self._remote_storage()._owner - if owner.cache_dir is not None: - return sum( - file.stat().st_size - for path in owner.cache_dir.iterdir() - if path.suffix in {".b2d", ".npy"} - for file in (path.rglob("*") if path.is_dir() else (path,)) - if file.is_file() - ) - return owner.cache_bytes - - @property - def metadata_bytes(self): - return self._remote_storage()._owner.arrow_metadata.serialized_size - - @property - def mutable(self): - return False - - @mutable.setter - def mutable(self, value): - raise RuntimeError("Parquet remote references are not supported") - - @property - def is_cache_mutable(self): - return True - - def save(self, *args, **kwargs): - destination = args[0] if args else kwargs.get("destination") - if destination is not None and str(destination).endswith(".b2z"): - return self._save_archive(*args, **kwargs) - return self._save_reference(*args, **kwargs) - - def _save_archive(self, destination, *, include_cache=True, overwrite=False, mutable=None): - from blosc2.remote_array import validate_persistable_url - - if mutable not in (None, False): - raise ValueError("Parquet references are read-only") - owner = self._remote_storage()._owner - if is_fsspec_url(owner.urlpath): - validate_persistable_url(owner.urlpath) - destination = Path(destination) - if destination.exists() and not overwrite: - raise FileExistsError(destination) - if owner.disk is not None and destination.resolve().is_relative_to(owner.disk.path.resolve()): - raise ValueError("destination cannot be inside the live cache") - if owner.source_marker is None: - owner.source_marker = _source_marker(owner.urlpath, owner.reopen_kwargs["storage_options"]) - destination.parent.mkdir(parents=True, exist_ok=True) - with ( - owner.lock, - _disk_guard(owner.disk), - tempfile.TemporaryDirectory(dir=destination.parent) as temporary, - ): - if owner.disk is None: - disk, root = _disk_cache_path( - owner.urlpath, - owner.reopen_kwargs["storage_options"], - owner.identity_options, - temporary, - owner.source_marker, - ) - try: - manifest = disk.load() - manifest["metadata"]["reopen"] = _reference_options(owner) - manifest["metadata"]["parquet_discovery"] = owner.discovery - disk.publish(manifest) - if include_cache: - for (number, physical), (table, _) in owner.cache.items(): - with table.copy(urlpath=root / _group_filename(number, physical)): - pass - finally: - disk.close() - else: - root = owner.cache_dir - manifest = owner.disk.load() - if manifest["metadata"].get("reopen") is None: - raise TypeError("Parquet conversion options cannot be saved in a portable archive") - staged = Path(temporary) / "reference.b2z" - with zipfile.ZipFile(staged, "w", zipfile.ZIP_STORED) as archive: - archive.write(root.parent / "active_generation.json", "active_generation.json") - archive.write(root / "embed.b2e", "embed.b2e") - if include_cache: - for file in root.rglob("*"): - if file.is_file() and not file.name.startswith(".") and file != root / "embed.b2e": - archive.write(file, f"{root.name}/{file.relative_to(root)}") - os.replace(staged, destination) - return str(destination) - - @classmethod - def _open_cache_artifact(cls, path, *, mode="r", storage_options=None, source_url=None): - if mode != "r": - raise ValueError("Parquet caches are read-only") - path = Path(path).resolve() - carrier = blosc2.blosc2_ext.open(str(path / "embed.b2e"), "r", 0) - manifest = carrier.vlmeta["b2remote_manifest"] - source = manifest["source"] - if source.get("kind") != "parquet": - raise ValueError("Cache is not a Parquet table") - if is_fsspec_url(source["urlpath"]): - from blosc2.remote_array import validate_persistable_url - - validate_persistable_url(source["urlpath"]) - options = manifest["metadata"].get("reopen") - if options is None: - raise ValueError("This Parquet cache needs the original conversion options to reopen") - options = dict(options) - policy = options.pop("null_policy") - _restore_compression_options(options) - urlpath = source_url or source["urlpath"] - table = cls( - urlpath, - cache_dir=path.parent.parent, - storage_options=storage_options, - _effective_null_policy=NullPolicy(**policy), - _refresh_marker=manifest["metadata"]["source_marker"], - **options, - ) - if table._remote_storage()._owner.cache_dir != path: - table.close() - raise ValueError("Parquet cache source or options no longer match this generation") - return table - - @classmethod - def _open_archive_artifact(cls, path, *, mode="r", storage_options=None, source_url=None): - from blosc2.remote_store_cache import StoreDiskCache, validate_generation - - manager = tempfile.TemporaryDirectory(prefix="parquet-reference-") - try: - with zipfile.ZipFile(path) as archive: - carrier = blosc2.schunk_from_cframe(archive.read("embed.b2e")) - manifest = carrier.vlmeta["b2remote_manifest"] - source = manifest["source"] - if source.get("kind") != "parquet": - raise ValueError("Cache archive is not a Parquet table") - validate_generation(manifest["generation"]) - generation = f"{manifest['generation']}.b2d" - cache_root = StoreDiskCache.path_for(manager.name, source) - cache_root.mkdir(parents=True) - for member in archive.infolist(): - name = member.filename - if ( - ( - name not in {"active_generation.json", "embed.b2e"} - and not name.startswith(generation + "/") - ) - or ".." in Path(name).parts - or member.is_dir() - or member.compress_type != zipfile.ZIP_STORED - ): - raise ValueError("Invalid Parquet cache archive member") - target = cache_root / (f"{generation}/embed.b2e" if name == "embed.b2e" else name) - target.parent.mkdir(parents=True, exist_ok=True) - with archive.open(member) as source_file, target.open("wb") as target_file: - shutil.copyfileobj(source_file, target_file) - table = cls._open_cache_artifact( - cache_root / generation, - mode=mode, - storage_options=storage_options, - source_url=source_url, - ) - table._remote_storage()._owner._cache_manager = manager - return table - except BaseException: - manager.cleanup() - raise - - def _save_reference(self, destination, *, include_cache=True, overwrite=False, mutable=None): - """Write a CFrame-backed source reference without runtime credentials.""" - from blosc2.b2objects import make_b2object_carrier, write_b2object_payload - from blosc2.remote_array import validate_persistable_url - - if mutable not in (None, False): - raise ValueError("Parquet references are read-only") - owner = self._remote_storage()._owner - url = owner.urlpath - if is_fsspec_url(url): - validate_persistable_url(url) - if owner.source_marker is None: - owner.source_marker = _source_marker(url, owner.reopen_kwargs["storage_options"]) - if "size" not in owner.source_marker or len(owner.source_marker) < 2: - raise ValueError("A portable Parquet reference requires an ETag or modification time") - options = _reference_options(owner) - payload = { - "kind": "remote_parquet", - "version": 1, - "urlpath": url, - "source_marker": owner.source_marker, - "options": options, - } - if include_cache: - retained = {} - with owner.lock, _disk_guard(owner.disk): - if owner.cache_dir is not None: - with tempfile.TemporaryDirectory() as cache_export: - for path in owner.cache_dir.iterdir(): - if path.suffix == ".npy": - retained[path.name] = path.read_bytes() - elif path.suffix == ".b2d": - exported = Path(cache_export) / f"{path.stem}.b2z" - with CTable.open(str(path)) as table, table.copy(urlpath=exported): - pass - retained[exported.name] = exported.read_bytes() - else: - with tempfile.TemporaryDirectory() as cache_export: - for (number, physical), (table, _) in owner.cache.items(): - path = ( - Path(cache_export) - / f"{number}-{hashlib.sha256(physical.encode()).hexdigest()[:16]}.b2z" - ) - with table.copy(urlpath=path): - pass - retained[path.name] = path.read_bytes() - payload["cache_files"] = retained - destination = Path(destination) - if destination.exists() and not overwrite: - raise FileExistsError(destination) - carrier = make_b2object_carrier("remote_parquet", (1,), np.uint8) - write_b2object_payload(carrier, payload) - destination.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(dir=destination.parent) as temporary: - staged = Path(temporary) / "reference.b2nd" - carrier.save(staged) - os.replace(staged, destination) - return str(destination) - - @classmethod - def _from_payload(cls, payload, *, storage_options=None, parquet_options=None): - from blosc2.remote_array import validate_persistable_url - - if ( - not isinstance(payload, dict) - or payload.get("kind") != "remote_parquet" - or payload.get("version") != 1 - ): - raise ValueError("Invalid Parquet reference") - url = payload.get("urlpath") - marker = payload.get("source_marker") - options = payload.get("options") - if not isinstance(url, str) or not isinstance(marker, dict) or not isinstance(options, dict): - raise ValueError("Invalid Parquet reference fields") - if is_fsspec_url(url): - validate_persistable_url(url) - if "storage_options" in options or "cache_dir" in options: - raise ValueError("Parquet references cannot contain runtime transport options") - if _source_marker(url, storage_options) != marker: - raise RuntimeError("Parquet reference source has changed; create a new reference") - options = dict(options) - policy = options.pop("null_policy", None) - if not isinstance(policy, dict): - raise ValueError("Parquet reference has no null policy") - _restore_compression_options(options) - saved_reader = options.get("parquet_options") or {} - if parquet_options is not None and parquet_options != saved_reader: - raise ValueError("Parquet reader option set differs from the saved reference") - retained = payload.get("cache_files", {}) - if not isinstance(retained, dict): - raise ValueError("Invalid retained Parquet cache") - manager = tempfile.TemporaryDirectory() if retained else None - try: - return cls( - url, - storage_options=storage_options, - _effective_null_policy=NullPolicy(**policy), - cache_dir=manager.name if manager is not None else None, - _seed_cache=retained, - _cache_manager=manager, - **options, - ) - except BaseException: - if manager is not None: - manager.cleanup() - raise - - @classmethod - def open_reference(cls, path, *, storage_options=None, parquet_options=None): - """Reopen a saved reference with replacement runtime transport options.""" - from blosc2.b2objects import read_b2object_payload - - raw = blosc2.blosc2_ext.open(os.fspath(path), "r", 0) - return cls._from_payload( - read_b2object_payload(raw), storage_options=storage_options, parquet_options=parquet_options - ) diff --git a/src/blosc2/remote_store.py b/src/blosc2/remote_store.py index 19186e5b0..fa53caa31 100644 --- a/src/blosc2/remote_store.py +++ b/src/blosc2/remote_store.py @@ -31,6 +31,18 @@ RESERVED_NAMES = {"embed.b2e", "__vlmeta__"} +def public_source_url(url): + parsed = urlsplit(url) + if not parsed.scheme: + return url + clean = urlunsplit((parsed.scheme, parsed.netloc.rsplit("@", 1)[-1], parsed.path, parsed.query, "")) + try: + validate_persistable_url(clean) + except ValueError: + clean = urlunsplit((parsed.scheme, parsed.netloc.rsplit("@", 1)[-1], parsed.path, "", "")) + return clean + + class CacheMiss(Exception): """A cached-only table operation needs unavailable source data.""" @@ -83,7 +95,18 @@ def get_zip_offsets(zip_path: str) -> dict[str, dict[str, int]]: if not name or ":" in name: raise ValueError("Invalid RemoteStore archive path") if name != "embed.b2e": - RemoteDiscovery._validate(name) + if name.startswith("parquet-groups/"): + parts = name.split("/") + if ( + len(parts) < 3 + or any(part in {"", ".", ".."} for part in parts) + or any(char in name for char in "\\\0\n\r\t") + ): + raise ValueError("Invalid Parquet cache archive path") + else: + RemoteDiscovery._validate(name) + if (info.external_attr >> 16) & 0o170000 == 0o120000: + raise ValueError("RemoteStore archives cannot contain symbolic links") if info.filename in offsets or info.flag_bits & 1: raise ValueError("Invalid RemoteStore archive member") f.seek(info.header_offset) @@ -119,7 +142,7 @@ def _resolve_hdf5_options(hdf5_index, private_index, source_format): class RemoteDiscovery: """Shared metadata and source resources, independent of browser presentation.""" - def __init__( + def __init__( # noqa: C901 self, urlpath, storage_options=None, @@ -141,6 +164,7 @@ def __init__( _refresh_source=False, _b2z_blob=None, _local_source=False, + _parquet_conversion=None, ): self.urlpath, dataset, self.format = parse_container_url(urlpath, dataset) if _source_format is not None: @@ -185,6 +209,10 @@ def __init__( self.notice = None self.archive = None self.zstore = None + self.parquet_file = None + self.parquet_handle = None + self.parquet_cache = None + self.parquet_conversion = _parquet_conversion or {} self.hdf5_blob = None self.hdf5_source_cache_path = None if _hdf5_blob is not None: @@ -198,7 +226,21 @@ def __init__( self.disk = None self.generation = manifest["generation"] if manifest else uuid.uuid4().hex self.metadata = manifest["metadata"] if manifest else {} + if self.format == "parquet" and manifest: + from blosc2.ctable import NullPolicy + from blosc2.remote_parquet import _restore_compression_options + + self.parquet_conversion = dict(self.metadata["parquet"].get("conversion", {})) + self.parquet_conversion["_effective_null_policy"] = NullPolicy( + **self.metadata["parquet"]["discovery"]["options"]["null_policy"] + ) + _restore_compression_options(self.parquet_conversion) + elif self.format == "parquet": + from blosc2.ctable import get_null_policy + + self.parquet_conversion.setdefault("_effective_null_policy", get_null_policy()) self.persist_metadata = persist_metadata + self.parquet_persistent = persist_metadata self._external_filesystem = _filesystem self.source_validator = _source_validator self.batch_validator = _batch_validator @@ -233,6 +275,10 @@ def __init__( self._open_b2z() elif self.format == "hdf5": self._open_hdf5(_hdf5_index) + elif self.format == "parquet": + from blosc2.remote_parquet import discover_parquet + + discover_parquet(self, self.parquet_conversion) else: self._open_zarr() self._check_node_limit() @@ -243,7 +289,11 @@ def __init__( self.close() raise - def _restore_manifest(self, manifest): + def _restore_manifest(self, manifest): # noqa: C901 + if self.format == "parquet": + from blosc2.remote_parquet import validate_parquet_metadata + + validate_parquet_metadata(manifest["metadata"]) for path, entry in manifest["nodes"].items(): self._validate(path) if ( @@ -288,6 +338,10 @@ def _restore_manifest(self, manifest): self.nodes[path] = (kind, self.hdf5_index["datasets"][path]) elif self.format == "zarr" and self.zstore is None: self._open_zarr() + elif self.format == "parquet": + from blosc2.remote_parquet import discover_parquet + + discover_parquet(self, self.parquet_conversion) self._check_node_limit() def _validate_hdf5_index(self): @@ -319,6 +373,9 @@ def save_manifest(self): "metadata": self.metadata, "caches": sorted(self.caches), "batch_caches": sorted(self.batch_caches), + "parquet_caches": ( + sorted(self.parquet_cache._cache_sizes) if self.parquet_cache is not None else [] + ), "cache_policy": getattr(self, "cache_policy", blosc2.CachePolicy.DISK).value, "max_cache_bytes": getattr(self, "max_cache_bytes", None), "mutable": getattr(self, "mutable", False), @@ -869,6 +926,8 @@ def check_open(): def load_ctable_attrs(self, table_path): """Load one table's user attributes without opening its data arrays.""" + if self.format == "parquet": + return {} if table_path in self.attrs: return dict(self.attrs[table_path]) member = "/".join(part for part in (table_path, "_vlmeta.b2f") if part) @@ -963,7 +1022,31 @@ def release(self): if not self._users: self.close() - def restore_caches(self, manifest): + def restore_caches(self, manifest): # noqa: C901 + if self.format == "parquet": + from blosc2.remote_parquet import ParquetCache + + if self.parquet_cache is not None: + self.parquet_cache.close() + if self.artifact_path is not None and self.disk is None and manifest: + self._cleanup_dir = tempfile.TemporaryDirectory(prefix="parquet-artifact-") + target = Path(self._cleanup_dir.name) / "parquet-groups" + target.mkdir() + members = set(manifest.get("parquet_caches", [])) + if self.artifact_offsets is not None: + with zipfile.ZipFile(self.artifact_path) as archive: + for info in archive.infolist(): + parts = PurePosixPath(info.filename).parts + if len(parts) >= 3 and parts[0] == "parquet-groups" and parts[1] in members: + archive.extract(info, self._cleanup_dir.name) + else: + for name in members: + source = Path(self.artifact_path) / "parquet-groups" / name + if source.is_dir(): + shutil.copytree(source, target / name) + self.parquet_artifact_dir = target + self.parquet_cache = ParquetCache(self) + return if manifest: self.restoring = True for path in manifest["caches"]: @@ -1004,6 +1087,11 @@ def restore_caches(self, manifest): self.cache_coordinator.enforce() self.restoring = False + def group(self, number, physical): + if self.format != "parquet" or self.parquet_cache is None: + raise RuntimeError("Parquet cache is unavailable") + return self.parquet_cache.group(number, physical) + def get_cache(self, source, *, seed=None): key = next(path for path, value in self.sources.items() if value is source) coordinator_key = f"{self.cache_namespace}:{key}" if self.cache_namespace else key @@ -1104,6 +1192,7 @@ def prepare_refresh(self, kind): _source_cache_dir=self.source_cache_dir, _refresh_source=True, _local_source=self.local_source, + _parquet_conversion=self.parquet_conversion, ) try: if replacement.nodes[replacement.root][0] != kind: @@ -1116,6 +1205,8 @@ def prepare_refresh(self, kind): replacement.mutable = self.mutable replacement.restoring = True replacement.disk = self.disk + if replacement.format == "parquet": + replacement.restore_caches(None) replacement.hdf5_source_cache_path = self.hdf5_source_cache_path if self.hdf5_source_cache_path is not None and replacement.hdf5_blob is None: # A large replacement has no retained bytes, but must invalidate @@ -1125,6 +1216,8 @@ def prepare_refresh(self, kind): ).hexdigest() return replacement except BaseException: + if replacement.disk is self.disk: + replacement.disk = None replacement.close() raise @@ -1154,6 +1247,15 @@ def _close_resources(self): if self.zstore is not None: self.zstore.close() self.zstore = None + if getattr(self, "parquet_cache", None) is not None: + self.parquet_cache.close() + self.parquet_cache = None + if getattr(self, "parquet_file", None) is not None: + self.parquet_file.close() + self.parquet_file = None + if getattr(self, "parquet_handle", None) is not None: + self.parquet_handle.close() + self.parquet_handle = None for source in self.sources.values(): if isinstance(source, blosc2.HDF5NDSource): source.close() @@ -1195,7 +1297,7 @@ def _export_metadata(self): metadata["ranges"] = ranges return metadata - def save_selection( + def save_selection( # noqa: C901 self, full_path, destination: str | os.PathLike, @@ -1205,8 +1307,10 @@ def save_selection( overwrite: bool = False, ) -> str: """Export the current store or subtree to a portable .b2z reference archive.""" - if self.local_source: + if self.local_source and self.format != "parquet": raise ValueError("local-source caches cannot be exported as portable RemoteStore references") + if not self.local_source: + validate_persistable_url(self.urlpath) if not isinstance(include_cache, bool): raise TypeError("include_cache must be a boolean") if mutable is not None and not isinstance(mutable, bool): @@ -1214,6 +1318,17 @@ def save_selection( dest_abs, dest_dir = self._validate_save_destination(destination, overwrite) with self.lock: + if self.format == "parquet" and self.parquet_source_marker is None: + from blosc2.remote_parquet import _source_marker + + self.parquet_source_marker = _source_marker( + self.urlpath, self.storage_options, self.filesystem + ) + if "size" not in self.parquet_source_marker or len(self.parquet_source_marker) < 2: + raise ValueError( + "A portable Parquet reference requires source size and a version marker" + ) + self.metadata["parquet"]["source_marker"] = self.parquet_source_marker effective_mutable = self.mutable if mutable is None else mutable if include_cache: retained = self.cache_coordinator.cache_bytes @@ -1240,6 +1355,22 @@ def save_selection( exported_batches = self._export_batch_caches(full_path, staging_dir) if include_cache else [] + exported_parquet = [] + if self.format == "parquet" and include_cache: + for name, (table, _) in self.parquet_cache.cache.items(): + filename = self.parquet_cache._filename(*name) + destination = Path(staging_dir) / "parquet-groups" / filename + destination.parent.mkdir(parents=True, exist_ok=True) + with table.copy(urlpath=destination): + pass + exported_parquet.append(filename) + if self.parquet_cache.cache_dir is not None: + for path in self.parquet_cache.cache_dir.glob("*-*.b2d"): + if path.name not in exported_parquet and path.is_dir(): + destination = Path(staging_dir) / "parquet-groups" / path.name + shutil.copytree(path, destination) + exported_parquet.append(path.name) + linked_exports = self._export_linked_stores(full_path, staging_dir) if include_cache else {} exported_manifest = { @@ -1253,6 +1384,7 @@ def save_selection( "metadata": metadata, "caches": sorted(exported_caches), "batch_caches": sorted(exported_batches), + "parquet_caches": sorted(exported_parquet), "cache_policy": self.cache_policy.value, "max_cache_bytes": self.max_cache_bytes, "mutable": effective_mutable, @@ -1358,9 +1490,15 @@ def _collect_export_nodes(self, group_full, include_cache): "dataset": group_full, "kind": self.format, } + if self.local_source: + exported_source["local"] = True fingerprint = storage_options_fingerprint(getattr(self, "storage_options", None)) if fingerprint: exported_source["storage_options"] = fingerprint + if self.format == "parquet": + from blosc2.remote_parquet import parquet_identity + + exported_source["options"] = parquet_identity(self.parquet_conversion) if prefix: exported_nodes = { k: (v[0], v[1] if v[0] in {"ctable", "remote_store", "unsupported"} else None) @@ -1459,7 +1597,15 @@ class RemoteStore(RemoteObject): @classmethod def _try_open_artifact( - cls, urlpath, dataset, storage_options, cache_policy, max_cache_bytes, cache_dir, allow_array_root + cls, + urlpath, + dataset, + storage_options, + cache_policy, + max_cache_bytes, + cache_dir, + allow_array_root, + allow_table_root, ): if not os.path.exists(urlpath): return None @@ -1475,11 +1621,13 @@ def _try_open_artifact( "max_cache_bytes": None if max_cache_bytes is CACHE_POLICY_DEFAULT else max_cache_bytes, "cache_dir": cache_dir, "_allow_array_root": allow_array_root, + "_as_store": True, + "allow_table_root": allow_table_root, } return cls._open_artifact(urlpath, **kwargs) @staticmethod - def _cache_source(urlpath, dataset, source_format, storage_options): + def _cache_source(urlpath, dataset, source_format, storage_options, parquet_conversion=None): base_url, root, kind = parse_container_url(urlpath, dataset) local = not urlsplit(base_url).scheme or bool(os.path.splitdrive(base_url)[0]) if local: @@ -1494,6 +1642,10 @@ def _cache_source(urlpath, dataset, source_format, storage_options): fingerprint = storage_options_fingerprint(storage_options) if fingerprint: source["storage_options"] = fingerprint + if source["kind"] == "parquet": + from blosc2.remote_parquet import parquet_identity + + source["options"] = parquet_identity(parquet_conversion or {}) return source @staticmethod @@ -1512,7 +1664,7 @@ def _validate_nested_storage_options(value): if value is not None and not callable(value) and not isinstance(value, dict): raise TypeError("nested_storage_options must be a mapping or callable") - def __init__( + def __init__( # noqa: C901 self, urlpath, *, @@ -1539,6 +1691,7 @@ def __init__( nested_storage_options=None, _b2z_blob=None, _allow_local_source=False, + _parquet_conversion=None, ): dataset = blosc2.core.resolve_dataset_path(dataset, path) if not isinstance(urlpath, (str, os.PathLike)): @@ -1546,10 +1699,20 @@ def __init__( urlpath = os.fspath(urlpath) hdf5_index, _source_format = _resolve_hdf5_options(hdf5_index, _hdf5_index, _source_format) artifact = self._try_open_artifact( - urlpath, dataset, storage_options, cache_policy, max_cache_bytes, cache_dir, _allow_array_root + urlpath, + dataset, + storage_options, + cache_policy, + max_cache_bytes, + cache_dir, + _allow_array_root, + allow_table_root, ) if artifact is not None: - self._attach(artifact._owner, artifact._path) + try: + self._attach(artifact._owner, artifact._path) + finally: + artifact.close() return if dataset is not None and not isinstance(dataset, str): raise TypeError("dataset must be a string") @@ -1562,11 +1725,13 @@ def __init__( and (not urlsplit(base_url).scheme or os.path.splitdrive(base_url)[0]) and os.path.exists(base_url) ) - if not local_source: - validate_persistable_url(base_url) - else: + if local_source: base_url = os.path.abspath(base_url) urlpath = base_url + elif ( + _source_format != "parquet" and parse_container_url(base_url)[2] != "parquet" + ) or cache_policy is blosc2.CachePolicy.DISK: + validate_persistable_url(base_url) local_hdf5 = local_source and _source_format == "hdf5" disk = None source_cache_path = source_cache_marker = None @@ -1574,7 +1739,9 @@ def __init__( if cache_policy is blosc2.CachePolicy.DISK: from blosc2.remote_store_cache import StoreDiskCache - source = self._cache_source(urlpath, dataset, _source_format, storage_options) + source = self._cache_source( + urlpath, dataset, _source_format, storage_options, _parquet_conversion + ) disk = StoreDiskCache(cache_dir, source) try: manifest = disk.load() if disk is not None else manifest @@ -1612,6 +1779,7 @@ def __init__( _source_cache_dir=cache_dir if disk is not None else None, _b2z_blob=_b2z_blob, _local_source=local_source, + _parquet_conversion=_parquet_conversion, ) manifest = owner.restored_manifest owner.attach_hdf5_source_cache(source_cache_path, source_cache_marker) @@ -1813,6 +1981,7 @@ def with_sparse_cache( # noqa: C901 _traffic=None, _source_format=None, _hdf5_index=None, + _parquet_conversion=None, ): """Attach an immutable remote hierarchy to a cache shared across processes. @@ -1830,14 +1999,24 @@ def with_sparse_cache( # noqa: C901 dataset = blosc2.core.resolve_dataset_path(dataset, path) limit = normalize_cache_limit(blosc2.CachePolicy.DISK, max_cache_bytes) - base, root, kind = parse_container_url(urlpath, dataset) + base, root, kind = parse_container_url(os.fspath(urlpath), dataset) if _source_format is not None: kind = _source_format - validate_persistable_url(base) + local_parquet = kind == "parquet" and not urlsplit(base).scheme + if local_parquet: + base = os.path.abspath(base) + else: + validate_persistable_url(base) source = {"urlpath": base, "dataset": (root or "").strip("/"), "kind": kind} + if local_parquet: + source["local"] = True fingerprint = storage_options_fingerprint(storage_options) if fingerprint: source["storage_options"] = fingerprint + if kind == "parquet": + from blosc2.remote_parquet import parquet_identity + + source["options"] = parquet_identity(_parquet_conversion or {}) disk = SharedStoreCache(runtime_cache_path, source) with disk.guard(): current = disk.load() @@ -1869,6 +2048,8 @@ def with_sparse_cache( # noqa: C901 _traffic=_traffic, _source_format=kind, _hdf5_index=_hdf5_index, + _parquet_conversion=_parquet_conversion, + _local_source=local_parquet, ) owner.disk = disk owner.shared = True @@ -1876,6 +2057,20 @@ def with_sparse_cache( # noqa: C901 owner.max_cache_bytes = limit owner.cache_coordinator = CacheCoordinator(limit) try: + if seed_manifest is not None and kind == "parquet": + names = set(seed_manifest.get("parquet_caches", [])) + target = disk.path / f"{owner.generation}.b2d" + if os.path.isdir(carrier): + for name in names: + shutil.copytree( + Path(carrier) / "parquet-groups" / name, target / "parquet-groups" / name + ) + else: + with zipfile.ZipFile(carrier) as archive: + for info in archive.infolist(): + parts = PurePosixPath(info.filename).parts + if len(parts) >= 3 and parts[0] == "parquet-groups" and parts[1] in names: + archive.extract(info, target) owner.restore_caches(current) if seed_manifest is not None: for key in seed_manifest["caches"]: @@ -2017,6 +2212,23 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 for file in folder.glob("*.chunk") if file.stem.isdigit() ) + groups = [] + if manifest["source"]["kind"] == "parquet": + from blosc2.remote_parquet import valid_group_name + + folder = disk.path / f"{manifest['generation']}.b2d" / "parquet-groups" + for name in manifest.get("parquet_caches", []): + if not valid_group_name(name): + raise ValueError("Invalid cached Parquet group name") + path = folder / name + if path.is_dir() and not path.is_symlink(): + groups.append( + ( + name, + path, + sum(file.stat().st_size for file in path.rglob("*") if file.is_file()), + ) + ) nested = [] nested_root = disk.parent / "nested" if nested_root.is_dir() and not nested_root.is_symlink(): @@ -2041,6 +2253,7 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 total = ( sum(size for _, _, size in leaves) + sum(size for _, _, size in batches) + + sum(size for _, _, size in groups) + sum(size for _, _, size in nested) ) removed = [] @@ -2061,6 +2274,12 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 file.unlink() removed.append((key, int(file.stem))) total -= size + for name, path, size in groups: + if total <= target_bytes or len(removed) >= max_chunks: + break + shutil.rmtree(path) + removed.append(("parquet", name)) + total -= size for parent, source, size in nested: if total <= target_bytes or len(removed) >= max_chunks: break @@ -2072,6 +2291,11 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 ) removed.extend((f"linked:{parent.name}:{key}", chunk) for key, chunk in evicted) total += remaining - size + if manifest["source"]["kind"] == "parquet" and any(key == "parquet" for key, _ in removed): + manifest["parquet_caches"] = [ + name for name in manifest.get("parquet_caches", []) if (folder / name).is_dir() + ] + disk.publish(manifest) return tuple(removed), total def read_cached(self, path, item=(), *, nchunk=None): @@ -2244,7 +2468,7 @@ def source(self): return { "kind": self._owner.format, "version": 1, - "urlpath": self._owner.urlpath, + "urlpath": public_source_url(self._owner.urlpath), "dataset": full, "assume_immutable": True, } @@ -2419,7 +2643,12 @@ def _load_artifact_manifest(cls, urlpath): source = manifest["source"] if not isinstance(source, dict) or not isinstance(source.get("urlpath"), str): raise ValueError("Invalid RemoteStore manifest source descriptor") - validate_persistable_url(source["urlpath"]) + if not ( + (source.get("kind") == "parquet" or os.path.isdir(urlpath)) + and source.get("local") + and os.path.isfile(source["urlpath"]) + ): + validate_persistable_url(source["urlpath"]) cls._validate_artifact_manifest(manifest) return manifest, artifact_offsets @@ -2433,12 +2662,27 @@ def _validate_artifact_manifest(manifest, _depth=0): # noqa: C901 source = manifest["source"] root = source.get("dataset", "") RemoteDiscovery._validate(root) - if source.get("kind") not in {"b2z", "hdf5", "zarr"}: + if source.get("kind") not in {"b2z", "hdf5", "zarr", "parquet"}: raise ValueError("Invalid RemoteStore source kind") for field in ("nodes", "attrs", "listed", "metadata"): if not isinstance(manifest.get(field), dict): raise ValueError(f"Invalid RemoteStore manifest {field}") nodes = manifest["nodes"] + if source.get("kind") == "parquet": + from blosc2.remote_parquet import validate_parquet_metadata + + required = {"urlpath", "dataset", "kind", "options"} + if ( + not required <= set(source) <= required | {"local", "storage_options"} + or not isinstance(source["options"], str) + or len(source["options"]) != 64 + or any(char not in "0123456789abcdef" for char in source["options"]) + or ("local" in source and source["local"] is not True) + ): + raise ValueError("Invalid Parquet source descriptor") + if root or set(nodes) != {""} or nodes[""][0] != "ctable": + raise ValueError("Parquet RemoteStore requires one root CTable") + validate_parquet_metadata(manifest["metadata"]) for path, entry in nodes.items(): RemoteDiscovery._validate(path) if ( @@ -2450,10 +2694,13 @@ def _validate_artifact_manifest(manifest, _depth=0): # noqa: C901 if entry[0] == "ctable": metadata = entry[1] if ( - source.get("kind") not in {"b2z", "hdf5"} + source.get("kind") not in {"b2z", "hdf5", "parquet"} or not isinstance(metadata, dict) or metadata.get("kind") not in {"ctable", b"ctable"} - or not isinstance(metadata.get("schema"), (str, bytes)) + or ( + source.get("kind") != "parquet" + and not isinstance(metadata.get("schema"), (str, bytes)) + ) or ( source.get("kind") == "hdf5" and ( @@ -2506,6 +2753,15 @@ def _validate_artifact_manifest(manifest, _depth=0): # noqa: C901 ) ): raise ValueError("Invalid cached RemoteStore batch") + parquet_caches = manifest.get("parquet_caches", []) + if not isinstance(parquet_caches, list) or (parquet_caches and source.get("kind") != "parquet"): + raise ValueError("Invalid cached Parquet groups") + from blosc2.remote_parquet import valid_group_name + + if any(not valid_group_name(name) for name in parquet_caches) or len(set(parquet_caches)) != len( + parquet_caches + ): + raise ValueError("Invalid cached Parquet group name") linked = manifest.get("linked", {}) if not isinstance(linked, dict): raise ValueError("Invalid nested RemoteStore artifacts") @@ -2548,6 +2804,7 @@ def _open_mutable_artifact(cls, urlpath, manifest, storage_options, cache_policy dataset=source_desc.get("dataset"), manifest=manifest, persist_metadata=True, + _local_source=source_desc.get("local", False), ) except BaseException: if cleanup_dir is not None: @@ -2609,6 +2866,7 @@ def _open_immutable_artifact( dataset=source_desc.get("dataset"), manifest=manifest, persist_metadata=False, + _local_source=source_desc.get("local", False), _traffic=_traffic, _filesystem=( _filesystem_resolver(source_desc["urlpath"]) if _filesystem_resolver is not None else None @@ -2665,7 +2923,7 @@ def _open_artifact(cls, urlpath, mode="r", **kwargs): if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") if cache_policy is blosc2.CachePolicy.NONE and ( - manifest.get("caches") or manifest.get("batch_caches") + manifest.get("caches") or manifest.get("batch_caches") or manifest.get("parquet_caches") ): raise ValueError( "Cannot reopen a warm RemoteStore artifact with CachePolicy.NONE; " @@ -2702,6 +2960,13 @@ def _open_artifact(cls, urlpath, mode="r", **kwargs): urlpath, manifest, artifact_offsets, storage_options, cache_policy, limit, cache_dir ) + if kwargs.get("_as_store"): + if owner.nodes[owner.root][0] == "ctable" and not kwargs.get("allow_table_root"): + owner.close() + raise ValueError("RemoteStore requires a group; use RemoteCTable for a table") + result = object.__new__(cls) + result._attach(owner, "") + return result return cls._select_artifact(owner, kwargs.get("dataset"), kwargs.get("max_concurrency")) @classmethod diff --git a/src/blosc2/remote_store_cache.py b/src/blosc2/remote_store_cache.py index 9efebca38..24e27379a 100644 --- a/src/blosc2/remote_store_cache.py +++ b/src/blosc2/remote_store_cache.py @@ -255,6 +255,15 @@ def __enter__(self): if owner.zstore is not None: owner.zstore.close() owner.zstore = None + if owner.parquet_cache is not None: + owner.parquet_cache.close() + owner.parquet_cache = None + if owner.parquet_file is not None: + owner.parquet_file.close() + owner.parquet_file = None + if owner.parquet_handle is not None: + owner.parquet_handle.close() + owner.parquet_handle = None owner.sources.clear() owner.source_descriptors.clear() owner.batch_caches.clear() diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index bdc34484c..234cb5d05 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1847,11 +1847,6 @@ def _open_special_store(urlpath, mode, offset, **kwargs): if "b2remote_store" in schunk_meta: if offset != 0: raise ValueError("Offset must be 0 for RemoteStore") - from blosc2.remote_parquet import open_parquet_cache_artifact - - parquet = open_parquet_cache_artifact(urlpath, mode=mode, **kwargs) - if parquet is not None: - return parquet from blosc2.remote_store import RemoteStore return RemoteStore._open_artifact(urlpath, mode=mode, **kwargs) @@ -2978,9 +2973,8 @@ def open( # noqa: C901 ): if kwargs.get("source_format") not in (None, "parquet"): raise ValueError("source_format conflicts with the .parquet suffix") - if mode != "r" or offset or dataset is not None: - raise ValueError("Parquet sources require mode='r', offset=0, and no dataset path") - from blosc2.remote_parquet import RemoteParquetCTable + if mode != "r" or offset or (dataset is not None and dataset.strip("/")): + raise ValueError("Parquet sources require mode='r', offset=0, and the root table") kwargs.pop("source_format", None) lazy = kwargs.pop("lazy", True) @@ -3007,7 +3001,7 @@ def open( # noqa: C901 reader = kwargs.pop("parquet_options", None) or {} with fsspec_open(str(urlpath), "rb", options) as handle: return blosc2.CTable.from_parquet(handle, **kwargs, **reader) - return RemoteParquetCTable(str(urlpath), **kwargs) + return blosc2.RemoteCTable(str(urlpath), dataset=dataset, source_format="parquet", **kwargs) _reject_table_buffer_options(kwargs) _validate_shared_cache_request(urlpath, shared_cache, kwargs) if isinstance(urlpath, blosc2.URLPath): diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index 8210cef1e..3e9e40aa9 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -29,18 +29,15 @@ def test_windows_cache_directory_uses_source_basename(): def test_source_url_preserves_non_sensitive_query(): + from blosc2.remote_store import public_source_url + assert ( - remote_parquet._source_url("https://example.com/source.parquet?version=2#section") + public_source_url("https://example.com/source.parquet?version=2#section") == "https://example.com/source.parquet?version=2" ) - with pytest.raises(ValueError, match="credential-like"): - remote_parquet._disk_cache_path( - "https://example.com/source.parquet?token=secret", - None, - (), - None, - {"size": "1", "etag": "revision"}, - ) + assert public_source_url("https://example.com/source.parquet?token=secret") == ( + "https://example.com/source.parquet" + ) @pytest.mark.parametrize( @@ -324,11 +321,11 @@ def test_invalid_source_options_fail_clearly(tmp_path): def test_portable_reference_reopens_and_detects_source_change(tmp_path): path = tmp_path / "source.parquet" - carrier = tmp_path / "source.b2nd" + carrier = tmp_path / "source.b2z" pq.write_table(pa.table({"id": pa.array([1, None, 3], type=pa.int8())}), path) with blosc2.null_policy(blosc2.NullPolicy(signed_int_strategy="max")): with blosc2.open(path) as remote: - remote.save(carrier) + remote.save(carrier, include_cache=False) with blosc2.open(carrier) as reopened: assert reopened.to_arrow().column("id").to_pylist() == [1, None, 3] with blosc2.RemoteCTable.open_reference(carrier) as reopened: @@ -344,13 +341,14 @@ def test_portable_reference_reopens_and_detects_source_change(tmp_path): ) assert output.strip() == "[1, None, 3]" pq.write_table(pa.table({"id": pa.array([4, 5], type=pa.int8())}), path) - with pytest.raises(RuntimeError, match="source has changed"): - blosc2.open(carrier) + with blosc2.open(carrier) as reopened: + with pytest.raises((RuntimeError, OSError), match="changed"): + reopened["id"][0] def test_reference_retains_cached_group_and_runtime_options(tmp_path): path = tmp_path / "source.parquet" - carrier = tmp_path / "source.b2nd" + carrier = tmp_path / "source.b2z" pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) with blosc2.open(path, storage_options={"auto_mkdir": True}) as remote: assert remote["id"][5] == 5 @@ -363,7 +361,7 @@ def test_reference_retains_cached_group_and_runtime_options(tmp_path): def test_reference_retains_flattened_row_map(tmp_path): path = tmp_path / "root.parquet" - carrier = tmp_path / "root.b2nd" + carrier = tmp_path / "root.b2z" source = pa.Table.from_arrays( [pa.array([[{"id": i}] for i in range(6)], type=pa.list_(pa.struct([("id", pa.int32())])))], names=[""], @@ -379,30 +377,17 @@ def test_reference_retains_flattened_row_map(tmp_path): assert reopened.traffic.requests == before -def test_reference_rejects_nonportable_reader_option(tmp_path): - path = tmp_path / "source.parquet" - pq.write_table(pa.table({"id": [1]}), path) - with blosc2.open(path) as remote: - remote._storage._owner.reopen_kwargs["parquet_options"] = {"decryption_properties": object()} - with pytest.raises(TypeError, match="Nonportable"): - remote.save(tmp_path / "source.b2nd") - - def test_reference_reader_options_are_frozen(tmp_path): path = tmp_path / "source.parquet" - carrier = tmp_path / "source.b2nd" + carrier = tmp_path / "source.b2z" pq.write_table(pa.table({"name": ["a", "b", "a"]}), path) with blosc2.open(path, parquet_options={"read_dictionary": ["name"]}) as remote: assert remote["name"][0] == "a" remote.save(carrier, include_cache=True) with blosc2.RemoteCTable.open_reference(carrier) as reopened: assert reopened.to_arrow().column("name").to_pylist() == ["a", "b", "a"] - with pytest.raises(ValueError, match="differs"): + with pytest.raises(TypeError, match="frozen"): blosc2.RemoteCTable.open_reference(carrier, parquet_options={"read_dictionary": []}) - with pytest.raises(ValueError, match="differ"): - blosc2.RemoteCTable.open_reference( - carrier, parquet_options={"read_dictionary": ["name"], "coerce_int96_timestamp_unit": "ms"} - ) def test_closed_handle_rejects_cached_column(tmp_path): @@ -574,27 +559,13 @@ def test_invalid_retained_metadata_is_rebuilt(tmp_path): with blosc2.open(path, cache_dir=cache_dir) as table: disk = table._remote_storage()._owner.disk manifest = disk.load() - manifest["metadata"]["parquet_discovery"]["schema"] = {"version": 999} + manifest["metadata"]["parquet"]["discovery"]["schema"] = {"version": 999} disk.publish(manifest) with blosc2.open(path, cache_dir=cache_dir) as rebuilt: assert rebuilt["x"][0] == 1 - assert rebuilt._remote_storage()._owner.disk.load()["metadata"]["parquet_discovery"]["version"] == 1 - - -def test_existing_cache_adds_local_marker_index(tmp_path, monkeypatch): - path = tmp_path / "source.parquet" - cache_dir = tmp_path / "cache" - pq.write_table(pa.table({"x": [1, 2]}), path) - with blosc2.open(path, cache_dir=cache_dir): - pass - index = next(cache_dir.glob(".parquet-marker-*.json")) - index.unlink() - with blosc2.open(path, cache_dir=cache_dir): - pass - assert index.is_file() - monkeypatch.setattr(remote_parquet, "_source_marker", lambda *args: pytest.fail("source was checked")) - with blosc2.open(path, cache_dir=cache_dir) as warm: - assert warm.nrows == 2 + assert ( + rebuilt._remote_storage()._owner.disk.load()["metadata"]["parquet"]["discovery"]["version"] == 1 + ) @pytest.mark.parametrize("disk", [False, True]) @@ -634,6 +605,30 @@ def checked_replace(source, destination): assert result.strip() == "1" +def test_archive_seeds_shared_cache(tmp_path): + path = tmp_path / "source.parquet" + artifact = tmp_path / "reference.b2z" + pq.write_table(pa.table({"x": [1, 2, 3, 4]}), path, row_group_size=2) + with blosc2.open(path) as table: + assert table["x"][0] == 1 + table.save(artifact) + manifest, _ = blosc2.RemoteStore._load_artifact_manifest(str(artifact)) + conversion = dict(manifest["metadata"]["parquet"]["conversion"]) + conversion["_effective_null_policy"] = blosc2.NullPolicy( + **manifest["metadata"]["parquet"]["discovery"]["options"]["null_policy"] + ) + with blosc2.RemoteStore.with_sparse_cache( + path, + tmp_path / "shared", + carrier=artifact, + _parquet_conversion=conversion, + ) as store: + with store[""] as table: + before = store.traffic.requests + assert table["x"][0] == 1 + assert store.traffic.requests == before + + def test_cache_generation_reopens_with_compression_options(tmp_path): path = tmp_path / "source.parquet" pq.write_table(pa.table({"x": [1, 2, 3]}), path) diff --git a/tests/test_remote_store.py b/tests/test_remote_store.py index 77d03423d..fd7265aa4 100644 --- a/tests/test_remote_store.py +++ b/tests/test_remote_store.py @@ -15,6 +15,96 @@ fsspec = pytest.importorskip("fsspec") +def test_parquet_table_root_uses_store_owner(tmp_path): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + path = tmp_path / "data.parquet" + pq.write_table(pa.table({"value": [1, 2, 3, 4]}), path, row_group_size=2) + with blosc2.RemoteStore(path, allow_table_root=True, _allow_local_source=True) as store: + assert store.kind("") == "ctable" + with store[""] as table: + assert table._storage._owner is store._owner + np.testing.assert_array_equal(table["value"][:], [1, 2, 3, 4]) + + +def test_parquet_shared_cache_export_and_refresh(tmp_path): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + path = tmp_path / "data.parquet" + pq.write_table(pa.table({"value": [1, 2, 3, 4]}), path, row_group_size=2) + url = "memory:///store-root.parquet" + fs = fsspec.filesystem("memory") + fs.pipe("/store-root.parquet", path.read_bytes()) + cache = tmp_path / "cache" + archive = tmp_path / "warm.b2z" + + def read(store): + with store[""] as table: + return table["value"][3] + + with blosc2.RemoteStore.with_sparse_cache(url, cache) as store: + table = store[""] + assert table._storage._owner is store._owner + assert store.read_cached_table(read) == (False, None) + assert table["value"][3] == 4 + assert store.read_cached_table(read) == (True, 4) + source = store._owner.disk.source + store.save(archive) + table.close() + + with blosc2.open(archive) as table: + before = table.traffic.requests + assert table["value"][3] == 4 + assert table.traffic.requests == before + + with blosc2.RemoteStore.with_sparse_cache(url, cache) as store: + before = store.traffic.requests + with store[""] as table: + assert table["value"][3] == 4 + assert store.traffic.requests == before + + removed, remaining = blosc2.RemoteStore.trim_sparse_cache(cache, source, 0) + assert removed + assert remaining == 0 + + with blosc2.RemoteStore.with_sparse_cache(url, cache) as store: + assert store.read_cached_table(read) == (False, None) + old = store[""] + pq.write_table(pa.table({"value": [5, 6, 7, 8]}), path, row_group_size=2) + fs.pipe("/store-root.parquet", path.read_bytes()) + store.refresh() + with pytest.raises(RuntimeError, match="stale"): + old["value"][3] + old.close() + with store[""] as table: + assert table["value"][3] == 8 + + +def test_parquet_standalone_refresh_replaces_cache(tmp_path, monkeypatch): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + path = tmp_path / "refresh.parquet" + pq.write_table(pa.table({"value": [1, 2]}), path) + with blosc2.RemoteCTable(path, cache_dir=tmp_path / "cache") as table: + old = table["value"] + assert old[0] == 1 + pq.write_table(pa.table({"value": [3, 4]}), path) + table.refresh() + with pytest.raises(RuntimeError, match=r"stale|closed"): + old[0] + assert table["value"][0] == 3 + import blosc2.remote_parquet as parquet + + def fail(*args): + raise OSError("discovery failed") + + with monkeypatch.context() as patch: + patch.setattr(parquet, "discover_parquet", fail) + with pytest.raises(OSError, match="discovery failed"): + table.refresh() + assert table["value"][0] == 3 + + @dataclasses.dataclass class NestedIndexedRow: value: int = blosc2.field(blosc2.int64(), chunks=(64,), blocks=(16,)) From 9cedd6ee2390a7185860a60dc88b69dea73c29b6 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 20:17:39 +0200 Subject: [PATCH 17/19] Fix Windows Parquet shared cache source identity --- src/blosc2/remote_store.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/blosc2/remote_store.py b/src/blosc2/remote_store.py index fa53caa31..9c423dd81 100644 --- a/src/blosc2/remote_store.py +++ b/src/blosc2/remote_store.py @@ -2002,7 +2002,9 @@ def with_sparse_cache( # noqa: C901 base, root, kind = parse_container_url(os.fspath(urlpath), dataset) if _source_format is not None: kind = _source_format - local_parquet = kind == "parquet" and not urlsplit(base).scheme + local_parquet = kind == "parquet" and ( + not urlsplit(base).scheme or bool(os.path.splitdrive(base)[0]) + ) if local_parquet: base = os.path.abspath(base) else: From d57feb617971a68566ab58f89dcf08a6042cfd85 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 25 Sep 2026 20:29:42 +0200 Subject: [PATCH 18/19] Avoid same-tick fsspec cache identity in Windows test --- tests/test_fsspec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index e10b08279..da5589a8e 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -17,6 +17,7 @@ import subprocess import sys import threading +import time import numpy as np import pytest @@ -1091,6 +1092,9 @@ def test_cached_dir_refetches_on_same_size_change(tmp_path): localdir = blosc2.core.localize_fsspec_url("memory://samesize.b2d", tmp_path) assert pathlib.Path(localdir, "a.bin").read_bytes() == b"A" * 100 + # MemoryFileSystem's identity uses a creation timestamp; Windows clocks can + # return the same timestamp for consecutive writes. + time.sleep(0.05) memfs.pipe_file("/samesize.b2d/a.bin", b"B" * 100) localdir = blosc2.core.localize_fsspec_url("memory://samesize.b2d", tmp_path) assert pathlib.Path(localdir, "a.bin").read_bytes() == b"B" * 100 From 742d2d10f54333af302488992f82f503ef66bd68 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 26 Sep 2026 08:49:01 +0200 Subject: [PATCH 19/19] Parallelize remote Parquet range reads --- doc/reference/remotectable.rst | 14 ++- src/blosc2/ctable_remote_read.py | 39 ++++---- src/blosc2/remote_parquet.py | 141 ++++++++++++++++++++++------ tests/ctable/test_remote_parquet.py | 96 ++++++++++++++++++- 4 files changed, 238 insertions(+), 52 deletions(-) diff --git a/doc/reference/remotectable.rst b/doc/reference/remotectable.rst index 864d44e64..66d34cf84 100644 --- a/doc/reference/remotectable.rst +++ b/doc/reference/remotectable.rst @@ -88,8 +88,18 @@ export ``.b2z`` references tied to the local source path. HTTP Parquet sources need byte-range support. A persistent cache also needs a source size and version marker, such as an ETag, modification time, or -Backblaze B2 file ID. The ``traffic`` counter measures file-handle reads; -transport buffering may make actual HTTP transfer different. +Backblaze B2 file ID. HTTP read-ahead is disabled by default to avoid fetching +unused bytes; an explicit ``storage_options={"cache_type": "bytes"}`` restores +fsspec buffering. The ``traffic`` counter counts fetched ranges; explicitly +enabled transport buffering may make actual HTTP transfer different. + +Multi-column selections, including table previews, fetch independent Parquet +column chunks concurrently using ``max_concurrency`` (8 by default). Decoding +and cache writes stay on the calling thread. ``row_buffer_bytes`` bounds each +wave of compressed responses; one physical field larger than the budget runs +alone. A physical field with nested leaves is fetched as one unit. Reads still +load whole column chunks for the selected row groups, even for a few rows. +Use ``max_concurrency=1`` for serial transport. See :doc:`Working with Remote Tables <../guides/remote_tables>` for column access, filtering, buffering, reference saving, and materialization examples. diff --git a/src/blosc2/ctable_remote_read.py b/src/blosc2/ctable_remote_read.py index 2d0387f97..ada14ca69 100644 --- a/src/blosc2/ctable_remote_read.py +++ b/src/blosc2/ctable_remote_read.py @@ -348,25 +348,26 @@ def column_values(table, names, positions, *, null_masks=None): # noqa: C901 if storage._owner.format == "parquet": with storage._owner.lock: storage._check_open() - result = {} - for name in names: - result[name] = ( - table._cols[name][positions] - if name in table.col_names - else table._fetch_col_at_positions_uncached(name, positions) - ) - if null_masks is not None and name in table.col_names: - mask = table._null_mask(name) - if mask is not None: - null_masks[name] = ~mask[positions] - elif name in table.col_names: - mask = table._null_mask(name) - if mask is not None: - missing = ~mask[positions] - if missing.any(): - result[name] = list(result[name]) - for index in np.flatnonzero(missing): - result[name][index] = None + + def reader(name): + if name not in table.col_names: + return table._fetch_col_at_positions_uncached(name, positions) + values = yield from table._cols[name].reads(positions) + mask = table._null_mask(name) + if mask is not None: + missing = ~(yield from mask.reads(positions)) + if null_masks is not None: + null_masks[name] = missing + elif missing.any(): + values = list(values) + for index in np.flatnonzero(missing): + values[index] = None + return values + + result, peak = run_reads( + ((name, reader(name)) for name in names), storage.max_concurrency, storage.row_buffer_bytes + ) + storage._peak_row_buffer_bytes = peak return result source_columns = _source_columns(table) with storage._owner.lock: diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py index b028a9f3d..235809dc6 100644 --- a/src/blosc2/remote_parquet.py +++ b/src/blosc2/remote_parquet.py @@ -10,7 +10,7 @@ import uuid import zipfile from collections import OrderedDict -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from dataclasses import asdict from enum import Enum from types import SimpleNamespace @@ -138,16 +138,30 @@ class _CountingHandle: def __init__(self, handle, traffic): self.handle = handle self.traffic = traffic + self.ranges = () + + @contextmanager + def buffered_ranges(self, ranges): + previous, self.ranges = self.ranges, ranges + try: + yield + finally: + self.ranges = previous def read(self, size=-1): + start = self.handle.tell() + for offset, data in self.ranges: + if size >= 0 and offset <= start and start + size <= offset + len(data): + self.handle.seek(start + size) + return data[start - offset : start - offset + size] data = self.handle.read(size) self.traffic.charge(len(data)) return data def readinto(self, buffer): - size = self.handle.readinto(buffer) - self.traffic.charge(size) - return size + data = self.read(len(buffer)) + buffer[: len(data)] = data + return len(data) def __getattr__(self, name): return getattr(self.handle, name) @@ -223,6 +237,8 @@ def _open_source_handle(urlpath, storage_options, marker, traffic, filesystem=No if marker and urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session") else {} ) + if urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session"): + kwargs["cache_type"] = (storage_options or {}).get("cache_type", "none") return _CountingHandle(fs.open(path, "rb", **kwargs), traffic) @@ -522,8 +538,80 @@ def _trim_cache(self, target_bytes, *, max_chunks=None): evicted.append(key) return tuple(evicted) + def _ensure_reader(self): + owner = self.owner + if owner.parquet_file is None: + import pyarrow.parquet as pq + + if owner.artifact_path is not None and owner.parquet_source_marker is not None: + marker = _source_marker(owner.urlpath, owner.storage_options, owner.filesystem) + if marker != owner.parquet_source_marker: + raise RuntimeError("Parquet reference source has changed") + owner.parquet_handle = _open_source_handle( + owner.urlpath, + owner.storage_options, + owner.parquet_source_marker, + owner.traffic, + owner.filesystem, + ) + try: + owner.parquet_file = pq.ParquetFile( + owner.parquet_handle, + metadata=owner.arrow_metadata, + **owner.parquet_reader_options, + ) + except BaseException: + owner.parquet_handle.close() + owner.parquet_handle = None + raise + + def group_reads(self, number, physical): + """Fetch one missing physical field; caller owns decoding and cache changes.""" + owner = self.owner + path = None if self.cache_dir is None else self.cache_dir / _group_filename(number, physical) + if (number, physical) in self.cache or (path is not None and path.exists()): + return [] + if owner.cache_coordinator.cached_only: + return [] + self._ensure_reader() + handle = owner.parquet_handle.handle + if not hasattr(handle, "fs"): + return [] + group = owner.arrow_metadata.row_group(number) + ranges = [] + for index in range(group.num_columns): + column = group.column(index) + name = column.path_in_schema + if physical and name != physical and not name.startswith(physical + "."): + continue + offset = min( + value + for value in (column.dictionary_page_offset, column.data_page_offset) + if value is not None and value >= 0 + ) + size = column.total_compressed_size + if size < 0 or (handle.size is not None and offset + size > handle.size): + raise ValueError("Parquet column range exceeds source bounds") + ranges.append((offset, size)) + + def fetch(): + answers = [] + for offset, size in ranges: + if hasattr(handle, "async_fetch_range"): + # HTTPFile's transport checks range support without reading an ignored full response. + data = handle._fetch_range(offset, offset + size) + else: + data = handle.fs.cat_file(handle.path, start=offset, end=offset + size) + owner.traffic.charge(len(data)) + if len(data) != size: + raise OSError("Incomplete Parquet column range") + answers.append((offset, data)) + return answers + + return (yield fetch, (), sum(size for _, size in ranges)) if ranges else [] + @contextmanager - def group(self, number, physical): # noqa: C901 + def group(self, number, physical): owner = self.owner with owner.lock: from blosc2.remote_store import CacheMiss @@ -554,30 +642,7 @@ def group(self, number, physical): # noqa: C901 pass if owner.cache_coordinator.cached_only: raise CacheMiss - if owner.parquet_file is None: - import pyarrow.parquet as pq - - if owner.artifact_path is not None and owner.parquet_source_marker is not None: - marker = _source_marker(owner.urlpath, owner.storage_options, owner.filesystem) - if marker != owner.parquet_source_marker: - raise RuntimeError("Parquet reference source has changed") - owner.parquet_handle = _open_source_handle( - owner.urlpath, - owner.storage_options, - owner.parquet_source_marker, - owner.traffic, - owner.filesystem, - ) - try: - owner.parquet_file = pq.ParquetFile( - owner.parquet_handle, - metadata=owner.arrow_metadata, - **owner.parquet_reader_options, - ) - except BaseException: - owner.parquet_handle.close() - owner.parquet_handle = None - raise + self._ensure_reader() arrow = owner.parquet_file.read_row_group(number, columns=[physical], use_threads=False) options = owner.parquet_options if physical == "" and not options["flatten_root"]: @@ -713,6 +778,14 @@ def __getitem__(self, key): return self._getitem_locked(key) def _getitem_locked(self, key): + from blosc2.ctable_remote_read import run_reads + + values, peak = run_reads(((self.name, self.reads(key)),), 1, self.storage.row_buffer_bytes) + self.storage._peak_row_buffer_bytes = peak + return values[self.name] + + def reads(self, key): + """Yield transport work while selection, decoding and caching stay serial.""" self.storage._check_open() if isinstance(key, tuple) and len(key) == 1: key = key[0] @@ -734,12 +807,20 @@ def _getitem_locked(self, key): selected = np.flatnonzero(groups == group) start = 0 if group == 0 else int(self.storage.row_ends[group - 1]) local = positions[selected] - start - with self.storage._owner.group(int(group), self.storage.physical[self.name]) as table: + owner = self.storage._owner + physical = self.storage.physical[self.name] + ranges = yield from owner.parquet_cache.group_reads(int(group), physical) + # A retained group may be available without opening the source handle. + with ExitStack() as stack: + if ranges: + stack.enter_context(owner.parquet_handle.buffered_ranges(ranges)) + table = stack.enter_context(owner.group(int(group), physical)) if self.mask: mask = table._null_mask(self.name) part = np.ones(len(local), dtype=bool) if mask is None else mask[local] else: part = table._cols[self.name][local] + del ranges for target, value in zip(selected, part, strict=True): values[int(target)] = value result = ( diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py index 3e9e40aa9..ea9544f87 100644 --- a/tests/ctable/test_remote_parquet.py +++ b/tests/ctable/test_remote_parquet.py @@ -816,7 +816,7 @@ def do_GET(self): thread.start() try: url = f"http://127.0.0.1:{server.server_port}/served.parquet?signature=test" - with blosc2.open(url, storage_options={"block_size": 4096, "cache_type": "none"}) as remote: + with blosc2.open(url, storage_options={"block_size": 4096}) as remote: assert counts["requests"] == 1 assert counts["heads"] == 1 assert "signature" not in remote.source["urlpath"] @@ -1001,3 +1001,97 @@ def test_oversized_memory_group_is_not_retained(tmp_path): before = remote.traffic.requests assert remote["x"][1] == 2 assert remote.traffic.requests > before + + +@pytest.mark.parametrize(("workers", "budget", "expected"), [(1, 1 << 20, 1), (3, 1 << 20, 3), (3, 1, 1)]) +@pytest.mark.parametrize( + "policy", [blosc2.CachePolicy.MEMORY, blosc2.CachePolicy.NONE, blosc2.CachePolicy.DISK] +) +def test_parquet_parallel_reads_are_bounded(tmp_path, monkeypatch, workers, budget, expected, policy): + from blosc2.ctable_remote_read import column_values + + source = pa.table({name: [0, None, 2, 3, 4, 5] for name in ("x", "y", "z")}) + path = tmp_path / "parallel.parquet" + pq.write_table(source, path, row_group_size=3) + options = ( + {"cache_dir": tmp_path / "cache"} if policy is blosc2.CachePolicy.DISK else {"cache_policy": policy} + ) + with blosc2.open(path, max_concurrency=workers, **options) as remote: + remote.row_buffer_bytes = budget + owner = remote._remote_read_storage()._owner + fs = owner.parquet_handle.handle.fs + original = fs.cat_file + lock = threading.Lock() + barrier = threading.Barrier(expected) + state = {"active": 0, "peak": 0, "calls": 0, "bytes": 0} + caller = threading.current_thread() + group = remote_parquet.ParquetCache.group + + def checked_group(self, *args): + assert threading.current_thread() is caller + return group(self, *args) + + def fetch(*args, **kwargs): + with lock: + state["active"] += 1 + state["peak"] = max(state["peak"], state["active"]) + state["calls"] += 1 + first_wave = state["calls"] <= expected + try: + if first_wave: + barrier.wait(timeout=5) + data = original(*args, **kwargs) + with lock: + state["bytes"] += len(data) + return data + finally: + with lock: + state["active"] -= 1 + + monkeypatch.setattr(fs, "cat_file", fetch) + monkeypatch.setattr(remote_parquet.ParquetCache, "group", checked_group) + before = (remote.traffic.requests, remote.traffic.nbytes) + values = column_values(remote, remote.col_names, np.array([5, 1, 0, 5])) + assert values == {name: [5, None, 0, 5] for name in remote.col_names} + assert state["peak"] == expected + requests = 12 if policy is blosc2.CachePolicy.NONE else 6 + assert remote.traffic.requests - before[0] == state["calls"] == requests + assert remote.traffic.nbytes - before[1] == state["bytes"] + assert owner.parquet_handle.ranges == () + peak = remote._remote_read_storage()._peak_row_buffer_bytes + if budget > 1: + assert peak <= budget + else: + assert peak <= max( + owner.arrow_metadata.row_group(i).column(j).total_compressed_size + for i in range(2) + for j in range(3) + ) + warm = remote.traffic.requests + assert column_values(remote, remote.col_names, np.array([5, 1, 0, 5])) == values + assert remote.traffic.requests == warm + (requests if policy is blosc2.CachePolicy.NONE else 0) + + +def test_parquet_parallel_transport_failure_can_retry(tmp_path, monkeypatch): + from blosc2.ctable_remote_read import column_values + + path = tmp_path / "retry.parquet" + pq.write_table(pa.table({"x": [1, 2], "y": [3, 4]}), path) + with blosc2.open(path) as remote: + owner = remote._remote_read_storage()._owner + fs = owner.parquet_handle.handle.fs + original = fs.cat_file + + def fail(*args, **kwargs): + raise OSError("transport failed") + + monkeypatch.setattr(fs, "cat_file", fail) + with pytest.raises(OSError, match="transport failed"): + column_values(remote, remote.col_names, np.array([0])) + assert owner.parquet_handle.ranges == () + monkeypatch.setattr(fs, "cat_file", lambda *args, **kwargs: b"short") + with pytest.raises(OSError, match="Incomplete Parquet"): + column_values(remote, remote.col_names, np.array([0])) + monkeypatch.setattr(fs, "cat_file", original) + values = column_values(remote, remote.col_names, np.array([0])) + assert {name: list(value) for name, value in values.items()} == {"x": [1], "y": [3]}