diff --git a/bench/remote_parquet_chicago_http_capacity_hint.jsonl b/bench/remote_parquet_chicago_http_capacity_hint.jsonl new file mode 100644 index 000000000..4aa2389bf --- /dev/null +++ b/bench/remote_parquet_chicago_http_capacity_hint.jsonl @@ -0,0 +1,5 @@ +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.203, "cold_reads": 1, "cold_bytes": 331590, "cold_heads": 0, "cold_seconds": 0.08, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32.parquet", "file_bytes": 686857703, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.021, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.083, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f64.parquet", "file_bytes": 687450036, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.023, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.085, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.066, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi.parquet", "file_bytes": 640173859, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 26, "open_bytes": 333139, "open_heads": 1, "open_seconds": 0.37, "cold_reads": 1, "cold_bytes": 6660483, "cold_heads": 0, "cold_seconds": 1.087, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.07, "cache_bytes": 12609432, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "company", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.205, "cold_reads": 1, "cold_bytes": 1831, "cold_heads": 0, "cold_seconds": 0.089, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.063, "cache_bytes": 1008, "warm_equal": true, "groups": 25, "arrow_equal": true} diff --git a/bench/remote_parquet_chicago_http_leaf_projection.jsonl b/bench/remote_parquet_chicago_http_leaf_projection.jsonl new file mode 100644 index 000000000..72efc73da --- /dev/null +++ b/bench/remote_parquet_chicago_http_leaf_projection.jsonl @@ -0,0 +1,5 @@ +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.193, "cold_reads": 1, "cold_bytes": 331590, "cold_heads": 0, "cold_seconds": 0.081, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32.parquet", "file_bytes": 686857703, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.029, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.087, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.061, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f64.parquet", "file_bytes": 687450036, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.025, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 0.089, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.066, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi.parquet", "file_bytes": 640173859, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 26, "open_bytes": 333139, "open_heads": 1, "open_seconds": 0.355, "cold_reads": 1, "cold_bytes": 363880, "cold_heads": 0, "cold_seconds": 0.073, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.066, "cache_bytes": 356304, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "company", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.189, "cold_reads": 1, "cold_bytes": 1831, "cold_heads": 0, "cold_seconds": 0.094, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.067, "cache_bytes": 1008, "warm_equal": true, "groups": 25, "arrow_equal": true} diff --git a/bench/remote_parquet_chicago_http_optimized.jsonl b/bench/remote_parquet_chicago_http_optimized.jsonl new file mode 100644 index 000000000..21df62ebc --- /dev/null +++ b/bench/remote_parquet_chicago_http_optimized.jsonl @@ -0,0 +1,5 @@ +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.188, "cold_reads": 1, "cold_bytes": 331590, "cold_heads": 0, "cold_seconds": 17.571, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 34695136, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32.parquet", "file_bytes": 686857703, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.022, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 17.638, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 34695136, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f64.parquet", "file_bytes": 687450036, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.016, "cold_reads": 1, "cold_bytes": 352939, "cold_heads": 0, "cold_seconds": 17.862, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.069, "cache_bytes": 34695136, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi.parquet", "file_bytes": 640173859, "rows": 24314322, "column": "trip.sec", "transport": "http", "open_reads": 26, "open_bytes": 333139, "open_heads": 1, "open_seconds": 0.364, "cold_reads": 1, "cold_bytes": 6660483, "cold_heads": 0, "cold_seconds": 1.083, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 12632079, "warm_equal": true, "groups": 25, "arrow_equal": true} +{"file": "chicago-taxi-flat-f32-cl20.parquet", "file_bytes": 581387271, "rows": 24314322, "column": "company", "transport": "http", "open_reads": 1, "open_bytes": 65536, "open_heads": 1, "open_seconds": 0.189, "cold_reads": 1, "cold_bytes": 1831, "cold_heads": 0, "cold_seconds": 0.819, "warm_reads": 0, "warm_bytes": 0, "warm_heads": 0, "warm_seconds": 0.068, "cache_bytes": 34695431, "warm_equal": true, "groups": 25, "arrow_equal": true} diff --git a/bench/remote_parquet_chicago_http_results.json b/bench/remote_parquet_chicago_http_results.json new file mode 100644 index 000000000..07c4dec62 --- /dev/null +++ b/bench/remote_parquet_chicago_http_results.json @@ -0,0 +1,61 @@ +{ + "date": "2026-09-25", + "method": "Each file served by bench/remote_parquet_files.py --http on 127.0.0.1 with byte-range responses. Reads and bytes count completed server GET responses and response bodies; HEAD requests are counted separately. Headers and TCP overhead are excluded. Default in-memory converted cache; fsspec block_size=4096, cache_type=none. Cold access reads the final row; warm access repeats it in the same process.", + "files": [ + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "trip.sec", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 32679571, "seconds": 1.253}, + "cold": {"gets": 1, "heads": 0, "bytes": 331590, "seconds": 17.688}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32.parquet", + "column": "trip.sec", + "file_bytes": 686857703, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 33057447, "seconds": 1.084}, + "cold": {"gets": 1, "heads": 0, "bytes": 352939, "seconds": 17.885}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f64.parquet", + "column": "trip.sec", + "file_bytes": 687450036, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 33068884, "seconds": 1.122}, + "cold": {"gets": 1, "heads": 0, "bytes": 352939, "seconds": 18.131}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "company", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"gets": 2, "heads": 2, "bytes": 32679571, "seconds": 1.308}, + "cold": {"gets": 1, "heads": 0, "bytes": 1831, "seconds": 0.813}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.068}, + "arrow_equal": true + }, + { + "file": "chicago-taxi.parquet", + "column": "trip.sec", + "file_bytes": 640173859, + "rows": 24314322, + "groups": 25, + "open": {"gets": 27, "heads": 2, "bytes": 671548047, "seconds": 3.909}, + "cold": {"gets": 1, "heads": 0, "bytes": 6660483, "seconds": 1.051}, + "warm": {"gets": 0, "heads": 0, "bytes": 0, "seconds": 0.067}, + "arrow_equal": true + } + ] +} diff --git a/bench/remote_parquet_chicago_results.json b/bench/remote_parquet_chicago_results.json new file mode 100644 index 000000000..8908c637f --- /dev/null +++ b/bench/remote_parquet_chicago_results.json @@ -0,0 +1,61 @@ +{ + "date": "2026-09-25", + "method": "Local file-handle read calls and returned bytes from bench/remote_parquet_files.py; these are not HTTP or S3 wire requests. Cold access reads the final row; warm access repeats it in the same process. Default memory cache; no disk cache.", + "files": [ + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "trip.sec", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 32679571, "seconds": 1.242}, + "cold": {"reads": 1, "bytes": 331590, "seconds": 17.69}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32.parquet", + "column": "trip.sec", + "file_bytes": 686857703, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 33057447, "seconds": 1.087}, + "cold": {"reads": 1, "bytes": 352939, "seconds": 17.802}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.066}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f64.parquet", + "column": "trip.sec", + "file_bytes": 687450036, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 33068884, "seconds": 1.099}, + "cold": {"reads": 1, "bytes": 352939, "seconds": 17.988}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.067}, + "arrow_equal": true + }, + { + "file": "chicago-taxi-flat-f32-cl20.parquet", + "column": "company", + "file_bytes": 581387271, + "rows": 24314322, + "groups": 25, + "open": {"reads": 2, "bytes": 32679571, "seconds": 1.339}, + "cold": {"reads": 1, "bytes": 1831, "seconds": 0.862}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.069}, + "arrow_equal": true + }, + { + "file": "chicago-taxi.parquet", + "column": "trip.sec", + "file_bytes": 640173859, + "rows": 24314322, + "groups": 25, + "open": {"reads": 27, "bytes": 671548047, "seconds": 3.932}, + "cold": {"reads": 1, "bytes": 6660483, "seconds": 1.077}, + "warm": {"reads": 0, "bytes": 0, "seconds": 0.067}, + "arrow_equal": true + } + ] +} diff --git a/bench/remote_parquet_files.py b/bench/remote_parquet_files.py new file mode 100644 index 000000000..d99f3dced --- /dev/null +++ b/bench/remote_parquet_files.py @@ -0,0 +1,194 @@ +"""Measure lazy reads of local Parquet files directly or through localhost HTTP.""" + +import argparse +import http.server +import json +import math +import threading +import time +from contextlib import contextmanager, nullcontext +from email.utils import formatdate +from pathlib import Path + +import pyarrow.compute as pc +import pyarrow.parquet as pq + +import blosc2 + + +@contextmanager +def local_http_server(path): + counts = [0, 0, 0] # GET requests, GET body bytes, HEAD requests. + lock = threading.Lock() + size = path.stat().st_size + + class Ranged(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def headers_for_file(self, length): + self.send_header("Content-Length", str(length)) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) + + def do_HEAD(self): + with lock: + counts[2] += 1 + self.send_response(200) + self.headers_for_file(size) + self.end_headers() + + def do_GET(self): + span = self.headers.get("Range") + first, last = 0, size - 1 + if span: + start, _, end = span.removeprefix("bytes=").partition("-") + if start: + first = int(start) + last = min(int(end), size - 1) if end else size - 1 + else: + first = max(0, size - int(end)) + if first > last: + self.send_response(416) + self.send_header("Content-Range", f"bytes */{size}") + self.end_headers() + return + self.send_response(206 if span else 200) + self.headers_for_file(last - first + 1) + if span: + self.send_header("Content-Range", f"bytes {first}-{last}/{size}") + self.end_headers() + sent = 0 + with path.open("rb") as source: + source.seek(first) + remaining = last - first + 1 + while remaining: + data = source.read(min(1 << 20, remaining)) + if not data: + break + self.wfile.write(data) + sent += len(data) + remaining -= len(data) + with lock: + counts[0] += 1 + counts[1] += sent + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Ranged) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + + def snapshot(): + with lock: + return tuple(counts) + + yield f"http://127.0.0.1:{server.server_port}/{path.name}", snapshot + finally: + server.shutdown() + server.server_close() + thread.join() + + +def measure(path, column=None, *, open_only=False, http=False): + path = Path(path) + context = local_http_server(path) if http else nullcontext((path, None)) + with context as (source, server_snapshot): + started = time.perf_counter() + options = {"block_size": 4096, "cache_type": "none"} if http else None + with blosc2.open(source, storage_options=options) as remote: + snapshot = server_snapshot or (lambda: (remote.traffic.requests, remote.traffic.nbytes, 0)) + opened = snapshot() + open_seconds = time.perf_counter() - started + column = column or remote.col_names[0] + if open_only: + return { + "file": path.name, + "file_bytes": path.stat().st_size, + "rows": len(remote), + "column": column, + "transport": "http" if http else "file", + "open_reads": opened[0], + "open_bytes": opened[1], + "open_heads": opened[2], + "open_seconds": round(open_seconds, 3), + } + started = time.perf_counter() + cold_value = remote[column][-1] + cold_seconds = time.perf_counter() - started + after_cold = snapshot() + cold = tuple(a - b for a, b in zip(after_cold, opened, strict=True)) + started = time.perf_counter() + warm_value = remote[column][-1] + warm_seconds = time.perf_counter() - started + warm = tuple(a - b for a, b in zip(snapshot(), after_cold, strict=True)) + result = { + "file": path.name, + "file_bytes": path.stat().st_size, + "rows": len(remote), + "column": column, + "transport": "http" if http else "file", + "open_reads": opened[0], + "open_bytes": opened[1], + "open_heads": opened[2], + "open_seconds": round(open_seconds, 3), + "cold_reads": cold[0], + "cold_bytes": cold[1], + "cold_heads": cold[2], + "cold_seconds": round(cold_seconds, 3), + "warm_reads": warm[0], + "warm_bytes": warm[1], + "warm_heads": warm[2], + "warm_seconds": round(warm_seconds, 3), + "cache_bytes": remote.cache_bytes, + "warm_equal": bool( + cold_value == warm_value + or (isinstance(cold_value, float) and math.isnan(cold_value) and math.isnan(warm_value)) + ), + } + parquet = pq.ParquetFile(path) + result["groups"] = parquet.num_row_groups + if column in parquet.schema_arrow.names: + last = parquet.read_row_group(parquet.num_row_groups - 1, columns=[column]) + expected = last.column(0)[-1].as_py() + elif parquet.schema_arrow.names == [""]: + root = parquet.read_row_group(parquet.num_row_groups - 1, columns=[""]).column(0).combine_chunks() + leaf = pc.list_flatten(root) + for part in column.split("."): + leaf = leaf.field(part) + expected = leaf[-1].as_py() + else: + expected = None + if expected is not None: + result["arrow_equal"] = bool( + cold_value == expected + or (isinstance(cold_value, float) and math.isnan(cold_value) and math.isnan(expected)) + ) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="+", type=Path) + parser.add_argument("--column") + parser.add_argument("--open-only", action="store_true") + parser.add_argument( + "--http", action="store_true", help="serve each file over localhost HTTP with byte ranges" + ) + args = parser.parse_args() + failed = False + for parquet_path in args.paths: + try: + print( + json.dumps( + measure(parquet_path, args.column, open_only=args.open_only, http=args.http), default=str + ), + flush=True, + ) + except Exception as error: + failed = True + print( + json.dumps({"file": parquet_path.name, "error": f"{type(error).__name__}: {error}"}), + flush=True, + ) + if failed: + raise SystemExit(1) diff --git a/bench/remote_parquet_synthetic_optimized.txt b/bench/remote_parquet_synthetic_optimized.txt new file mode 100644 index 000000000..70dd378b4 --- /dev/null +++ b/bench/remote_parquet_synthetic_optimized.txt @@ -0,0 +1,5 @@ +scalar: file=109920 B open=1 reads/65536 B/0.004 s cold=1 reads/5364 B/0.108 s warm=0 reads/0 B/0.001 s cache=138536 B python_peak=386880 B +wide: file=1096185 B open=1 reads/65536 B/0.026 s cold=1 reads/5364 B/0.118 s warm=0 reads/0 B/0.002 s cache=138536 B python_peak=275013 B +strings: file=64961 B open=1 reads/64961 B/0.001 s cold=1 reads/6352 B/0.007 s warm=0 reads/0 B/0.002 s cache=67367 B python_peak=132149 B +dictionary: file=8231 B open=1 reads/8231 B/0.001 s cold=1 reads/701 B/0.006 s warm=0 reads/0 B/0.001 s cache=134786 B python_peak=304604 B +unnamed-root: file=55530 B open=11 reads/109198 B/0.003 s cold=1 reads/5368 B/0.003 s warm=0 reads/0 B/0.001 s cache=3283 B python_peak=8441705 B diff --git a/bench/remote_parquet_traffic.py b/bench/remote_parquet_traffic.py new file mode 100644 index 000000000..88d346323 --- /dev/null +++ b/bench/remote_parquet_traffic.py @@ -0,0 +1,62 @@ +"""Count in-memory Parquet read operations and bytes for representative tables.""" + +import io +import time +import tracemalloc + +import fsspec +import pyarrow as pa +import pyarrow.parquet as pq + +import blosc2 + + +def measure(name, table, row_group_size, column): + output = io.BytesIO() + pq.write_table(table, output, row_group_size=row_group_size) + blob = output.getvalue() + url = f"memory:///remote-parquet-{name}.parquet" + fsspec.filesystem("memory").pipe(f"/remote-parquet-{name}.parquet", blob) + tracemalloc.start() + started = time.perf_counter() + with blosc2.open(url) as remote: + open_time = time.perf_counter() - started + _, open_peak = tracemalloc.get_traced_memory() + tracemalloc.reset_peak() + opened = (remote.traffic.requests, remote.traffic.nbytes) + started = time.perf_counter() + remote[column][-1] + cold_time = time.perf_counter() - started + _, cold_peak = tracemalloc.get_traced_memory() + tracemalloc.reset_peak() + cold = (remote.traffic.requests - opened[0], remote.traffic.nbytes - opened[1]) + cache_bytes = remote.cache_bytes + started = time.perf_counter() + remote[column][-1] + warm_time = time.perf_counter() - started + warm = (remote.traffic.requests - opened[0] - cold[0], remote.traffic.nbytes - opened[1] - cold[1]) + tracemalloc.stop() + print( + f"{name}: file={len(blob)} B open={opened[0]} reads/{opened[1]} B/{open_time:.3f} s " + f"cold={cold[0]} reads/{cold[1]} B/{cold_time:.3f} s " + f"warm={warm[0]} reads/{warm[1]} B/{warm_time:.3f} s " + f"cache={cache_bytes} B python_peak={max(open_peak, cold_peak)} B" + ) + + +if __name__ == "__main__": + n = 10_000 + measure("scalar", pa.table({"x": list(range(n)), "y": list(range(n))}), 1_000, "x") + measure("wide", pa.table({f"c{i}": list(range(n)) for i in range(20)}), 1_000, "c0") + measure("strings", pa.table({"s": [f"value-{i:05d}" for i in range(n)]}), 1_000, "s") + measure( + "dictionary", + pa.table({"d": pa.array([f"v{i % 100}" for i in range(n)]).dictionary_encode()}), + 1_000, + "d", + ) + root = pa.Table.from_arrays( + [pa.array([[{"x": i}] for i in range(n)], type=pa.list_(pa.struct([("x", pa.int64())])))], + names=[""], + ) + measure("unnamed-root", root, 1_000, "x") diff --git a/doc/guides/remote_tables.md b/doc/guides/remote_tables.md index 31742d083..50332f3f3 100644 --- a/doc/guides/remote_tables.md +++ b/doc/guides/remote_tables.md @@ -5,6 +5,59 @@ Fixed-width, `blosc2.utf8()`, batch-backed variable-length, list, struct/object, and dictionary columns are fetched on demand, including their null masks. A table inside a hierarchy can also be opened through `RemoteStore`. +## Single-file Parquet + +`blosc2.open()` recognizes `.parquet` paths, including fsspec URLs, and returns +a read-only `RemoteCTable`. Use `source_format="parquet"` for an extensionless +URL. `storage_options` go to fsspec; `parquet_options` go to PyArrow's +`ParquetFile`. `columns`, `max_rows`, null policy, string width, and list +conversion options follow `CTable.from_parquet()`. + +```python +with blosc2.open( + "s3://bucket/readings.parquet", + storage_options={"anon": True}, + columns=["station", "temperature"], + parquet_options={"read_dictionary": ["station"]}, + cache_dir="parquet-cache", +) as table: + last = table["temperature"][-1] + local = table.copy(urlpath="readings.b2z") +``` + +Reads fetch a Parquet field one row group at a time. Selecting one column avoids +unrelated columns; reading a complete row may fetch every column in its group. +`cache_dir` retains converted data and table metadata across opens, so warm +opens and cached reads need no connection to the source. Cached sources are +assumed unchanged until `refresh()` is called. Use `lazy=False` to import the +whole table eagerly. + +Parquet is a `RemoteStore` with one root CTable. Use +`RemoteStore(url, allow_table_root=True)` and `store[""]` when a store operation +needs to own the cache and traffic counters. A Parquet file accepts only the +root selector (`""` or `"/"`); selecting a child path raises an error. +`RemoteCTable(url)` and lazy `blosc2.open(url)` use the same owner. NONE retains +no converted row groups, MEMORY shares the store budget, and DISK retains +complete converted physical-column/row-group units. A small slice can therefore +read a whole row group. Shared DISK caches support `read_cached_table()` and +offline `trim_sparse_cache()` with the same aggregate allowance. +Caches from the earlier Parquet prototype layout are not migrated; use a fresh +`cache_dir` for this RemoteStore layout. + +HTTP servers must honor byte-range requests; fsspec raises a range-request error +for servers that only return complete files. Download the file and import it +locally when range access is unavailable. + +`table.save("reference.b2z")` creates a RemoteStore archive with data already in +the cache. Opening it reuses that data and fetches missing groups on demand. +Archives of local sources keep an absolute path to the local Parquet file. +Use `include_cache=False` to save only the table metadata. `.b2z` exports use +the RemoteStore version 1 manifest with `kind="parquet"`. Runtime filesystems, credentials, +and storage options must be supplied again when reopening. + +Use `shared_cache=True` when separate processes need to share one `cache_dir`. +They should use the same source and conversion options. + `blosc2.open()` dispatches uncached local B2Z table archives to `CTable`, and remote B2Z archives and selected local or remote PyTables tables to `RemoteCTable`. Supplying `cache_dir=` also selects `RemoteCTable` for a local @@ -126,6 +179,12 @@ See `examples/ctable/remote_handling.py` for a batched archive writer with nulla multilingual UTF-8 and variable-length strings, a batch-backed list, and a dictionary. It reports ordinary batch cold/warm reads and dictionary code/vocabulary costs separately. +The same script writes and reads Blosc2 with `--blosc2`, PyTables/HDF5 with +`--pytables`, and Parquet with `--parquet`. Without a format flag, it selects the +format from the `.b2z`, `.h5`, or `.parquet` extension of the output path or input URL. +Use `--write FILE` to create a file locally, then pass its uploaded URL to read it remotely. +The Parquet file uses the full Blosc2 example schema, including nullable values, +multilingual strings, lists, and dictionary-encoded regions. ## Refresh a remote table diff --git a/doc/reference/remotectable.rst b/doc/reference/remotectable.rst index 22ea13dda..66d34cf84 100644 --- a/doc/reference/remotectable.rst +++ b/doc/reference/remotectable.rst @@ -4,8 +4,8 @@ RemoteCTable ============ ``RemoteCTable`` is a read-only :class:`blosc2.CTable` backed by a remote B2Z -archive or a local or remote PyTables/HDF5 table. Fixed-width, shaped, nullable, -UTF-8, batch-backed variable-length, +archive, a local or remote PyTables/HDF5 table, or a Parquet file. Fixed-width, +shaped, nullable, UTF-8, batch-backed variable-length, batch-backed list, struct/object, and dictionary columns are fetched on demand. Open local PyTables tables through :func:`blosc2.open` with ``path=`` or a ``::table`` selector. @@ -74,6 +74,33 @@ peak RAM. for advanced attachment with manifests or seed carriers, with the same default budget and shared-cache implementation. +Parquet cache details +--------------------- + +Parquet ``cache_dir`` stores each accessed physical field and row group as a +native CTable directory under the RemoteStore generation's ``parquet-groups`` +directory. The common manifest retains the footer, schema, conversion options, +source marker, and row-group boundaries. Warm opens reuse this discovery and +payload without contacting the source. Cached sources are assumed immutable +until ``refresh()`` is called. New portable ``.b2z`` archives use the common +RemoteStore manifest. Local Parquet sources use the same cache layout and may +export ``.b2z`` references tied to the local source path. + +HTTP Parquet sources need byte-range support. A persistent cache also needs a +source size and version marker, such as an ETag, modification time, or +Backblaze B2 file ID. HTTP read-ahead is disabled by default to avoid fetching +unused bytes; an explicit ``storage_options={"cache_type": "bytes"}`` restores +fsspec buffering. The ``traffic`` counter counts fetched ranges; explicitly +enabled transport buffering may make actual HTTP transfer different. + +Multi-column selections, including table previews, fetch independent Parquet +column chunks concurrently using ``max_concurrency`` (8 by default). Decoding +and cache writes stay on the calling thread. ``row_buffer_bytes`` bounds each +wave of compressed responses; one physical field larger than the budget runs +alone. A physical field with nested leaves is fetched as one unit. Reads still +load whole column chunks for the selected row groups, even for a few rows. +Use ``max_concurrency=1`` for serial transport. + See :doc:`Working with Remote Tables <../guides/remote_tables>` for column access, filtering, buffering, reference saving, and materialization examples. diff --git a/doc/reference/remotestore.rst b/doc/reference/remotestore.rst index 904f3d11f..b5c7884f9 100644 --- a/doc/reference/remotestore.rst +++ b/doc/reference/remotestore.rst @@ -3,7 +3,7 @@ RemoteStore =========== -``RemoteStore`` discovers a read-only B2Z, Zarr or HDF5 hierarchy and returns +``RemoteStore`` discovers a read-only B2Z, Zarr, HDF5 or Parquet source and returns :ref:`RemoteArray` and :ref:`RemoteCTable` leaves. Groups and leaves share one source session: a B2Z archive, a native HDF5 index, or a Zarr store. Zarr listing remains lazy. @@ -16,6 +16,11 @@ DISK accepts ``max_cache_bytes=None`` for unbounded retention. Sources must be immutable. Generic ``blosc2.open(..., lazy=True, dataset=...)`` continues to open a single array. +A Parquet file has one CTable at its root. Pass ``allow_table_root=True`` to +retain a store handle, then use ``store[""]`` to access the table. Parquet has no +child selectors. Converted physical-column/row-group caches share the store's +policy, allowance, traffic counter, persistence, and refresh generation. + For HDF5, ``hdf5_index=`` accepts a native index dictionary, local JSON path, or remote fsspec URL. An explicit index skips hierarchy discovery and must match the source URL and selected scope. See :doc:`../guides/remote_arrays` for the diff --git a/examples/ctable/remote_handling.py b/examples/ctable/remote_handling.py index 71355e385..93e65e475 100644 --- a/examples/ctable/remote_handling.py +++ b/examples/ctable/remote_handling.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Create or access a CTable with fixed-width, UTF-8, batch-backed and dictionary columns.""" +"""Create or access remote-friendly Blosc2, PyTables, and Parquet tables.""" import argparse import pprint @@ -14,6 +14,7 @@ import time from dataclasses import dataclass, fields from pathlib import Path +from urllib.parse import urlsplit import numpy as np @@ -40,15 +41,38 @@ class Reading: FULL_INDEX_UNSUPPORTED = {"message", "tags"} +TABLE_NAME = "readings" def make_notes(ids): notes = np.array(["", "café", "東京の観測", "🌦️ weather improving"], dtype=object)[ids % 4] - notes = np.array( + return np.array( [f"{text} #{i}" if text else "" for i, text in zip(ids, notes, strict=True)], dtype=object ) - notes[ids % 43 == 0] = None - return notes + + +def rich_batches(rows, batch_size): + rng = np.random.default_rng(42) + statuses = np.array(["ok", "warning", "offline", ""], dtype=object) + for start in range(0, rows, batch_size): + ids = np.arange(start, min(start + batch_size, rows), dtype=np.int64) + temperature = rng.normal(18, 10, len(ids)).astype(np.float32) + humidity = rng.integers(0, 101, len(ids), dtype=np.int16) + status = statuses[(ids // 7) % len(statuses)] + regions = np.array(["north", "south", "east", "west"], dtype=object)[ids % 4] + regions[ids % 59 == 0] = None + yield { + "id": ids, + "station_id": (ids % 100).astype(np.int32), + "temperature": np.ma.array(temperature, mask=ids % 17 == 0), + "humidity": np.ma.array(humidity, mask=ids % 29 == 0), + "status": np.ma.array(status, mask=ids % 41 == 0), + "active": ids % 5 != 0, + "note": np.ma.array(make_notes(ids), mask=ids % 43 == 0), + "message": [None if i % 47 == 0 else f"sensor {i}: café 東京" for i in ids], + "tags": [None if i % 53 == 0 else [int(i % 7), int(i % 11)] for i in ids], + "region": regions, + } def write_table(args) -> None: @@ -77,8 +101,6 @@ def write_table(args) -> None: raise ValueError("--full columns must not contain duplicates") output.parent.mkdir(parents=True, exist_ok=True) - rng = np.random.default_rng(42) - statuses = np.array(["ok", "warning", "offline", ""], dtype=object) attrs = { "version": 1, "sampling_interval": 0.5, @@ -96,34 +118,8 @@ def write_table(args) -> None: ) as table: for name, value in attrs.items(): table.attrs[name] = value - for start in range(0, args.rows, args.batch_size): - stop = min(start + args.batch_size, args.rows) - ids = np.arange(start, stop, dtype=np.int64) - temperature = rng.normal(18, 10, len(ids)).astype(np.float32).astype(object) - humidity = rng.integers(0, 101, len(ids), dtype=np.int16).astype(object) - status = statuses[(ids // 7) % len(statuses)].copy() - temperature[ids % 17 == 0] = None - humidity[ids % 29 == 0] = None - status[ids % 41 == 0] = None - messages = [None if i % 47 == 0 else f"sensor {i}: café 東京" for i in ids] - tags = [None if i % 53 == 0 else [int(i % 7), int(i % 11)] for i in ids] - regions = np.array(["north", "south", "east", "west"], dtype=object)[ids % 4] - regions[ids % 59 == 0] = None - table.extend( - { - "id": ids, - "station_id": (ids % 100).astype(np.int32), - "temperature": temperature, - "humidity": humidity, - "status": status, - "active": ids % 5 != 0, - "note": make_notes(ids), - "message": messages, - "tags": tags, - "region": regions, - }, - validate=False, - ) + for batch in rich_batches(args.rows, args.batch_size): + table.extend(batch, validate=False) if indexed_columns: started = time.perf_counter() for name in indexed_columns: @@ -149,6 +145,134 @@ def write_table(args) -> None: print(f"Now upload {output} to your cloud object storage.") +def simple_batches(rows, batch_size): + rng = np.random.default_rng(42) + statuses = np.array(["ok", "warning", "offline", ""]) + notes = np.array(["", "clear", "cloudy", "rain"]) + for start in range(0, rows, batch_size): + ids = np.arange(start, min(start + batch_size, rows), dtype=np.int64) + yield { + "id": ids, + "station_id": (ids % 100).astype(np.int32), + "temperature": rng.normal(18, 10, len(ids)).astype(np.float32), + "humidity": rng.integers(0, 101, len(ids), dtype=np.int16), + "status": statuses[(ids // 7) % len(statuses)], + "active": ids % 5 != 0, + "note": notes[ids % len(notes)], + } + + +def write_pytables(args) -> None: + import tables + + class PyTablesReading(tables.IsDescription): + id = tables.Int64Col(pos=0) + station_id = tables.Int32Col(pos=1) + temperature = tables.Float32Col(pos=2) + humidity = tables.Int16Col(pos=3) + status = tables.StringCol(8, pos=4) + active = tables.BoolCol(pos=5) + note = tables.StringCol(32, pos=6) + + output = args.write + if output.suffix != ".h5": + raise ValueError("output must end in .h5") + if args.rows < 1 or args.batch_size < 1: + raise ValueError("--rows and --batch-size must be positive") + if output.exists() and not args.overwrite: + raise FileExistsError(f"{output} already exists; pass --overwrite to replace it") + indexed_columns = [] + if args.full is not None: + indexed_columns = ( + list(PyTablesReading.columns) + if args.full == "*" + else [name.strip() for name in args.full.split(",")] + ) + unknown = set(indexed_columns) - set(PyTablesReading.columns) + if not all(indexed_columns) or unknown: + raise ValueError(f"invalid --full columns: {', '.join(sorted(unknown)) or args.full!r}") + if len(indexed_columns) != len(set(indexed_columns)): + raise ValueError("--full columns must not contain duplicates") + + output.parent.mkdir(parents=True, exist_ok=True) + with tables.open_file(output, mode="w") as h5file: + filters = tables.Filters(complevel=5, complib="blosc2:zstd", shuffle=True) + table = h5file.create_table( + "/", + TABLE_NAME, + PyTablesReading, + title="Synthetic weather-station readings", + filters=filters, + expectedrows=args.rows, + ) + table.attrs.version = 1 + table.attrs.sampling_interval = 0.5 + table.attrs.description = "Synthetic weather-station readings" + + for batch in simple_batches(args.rows, args.batch_size): + data = np.empty(len(batch["id"]), dtype=table.dtype) + for name, values in batch.items(): + data[name] = values + table.append(data) + table.flush() + + if indexed_columns: + started = time.perf_counter() + for name in indexed_columns: + getattr(table.cols, name).create_csindex(filters=filters) + print(f"Created FULL (CSI) indexes in {time.perf_counter() - started:.2f} s") + + with tables.open_file(output) as h5file: + table = h5file.root.readings + assert table.nrows == args.rows + assert all(getattr(table.cols, name).index.is_csi for name in indexed_columns) + + print(f"Created {output} ({output.stat().st_size / 1_000_000:.1f} MB, {args.rows:,} rows)") + print(f"FULL (CSI) indexes: {', '.join(indexed_columns) if indexed_columns else 'none'}") + print(f"Now upload {output} to your cloud object storage.") + + +def write_parquet(args) -> None: + import pyarrow as pa + import pyarrow.parquet as pq + + output = args.write + if output.suffix != ".parquet": + raise ValueError("output must end in .parquet") + if args.rows < 1 or args.batch_size < 1: + raise ValueError("--rows and --batch-size must be positive") + if output.exists() and not args.overwrite: + raise FileExistsError(f"{output} already exists; pass --overwrite to replace it") + + output.parent.mkdir(parents=True, exist_ok=True) + schema = pa.schema( + [ + ("id", pa.int64()), + ("station_id", pa.int32()), + ("temperature", pa.float32()), + ("humidity", pa.int16()), + ("status", pa.string()), + ("active", pa.bool_()), + ("note", pa.large_string()), + ("message", pa.large_string()), + ("tags", pa.list_(pa.int16())), + ("region", pa.dictionary(pa.int32(), pa.string())), + ] + ) + with pq.ParquetWriter( + output, + schema, + compression="zstd", + use_dictionary=["station_id", "humidity", "status", "tags.list.element", "region"], + ) as writer: + for batch in rich_batches(args.rows, args.batch_size): + writer.write_table(pa.table(batch, schema=schema), row_group_size=args.batch_size) + + assert pq.read_metadata(output).num_rows == args.rows + print(f"Created {output} ({output.stat().st_size / 1_000_000:.1f} MB, {args.rows:,} rows)") + print(f"Now upload {output} to your cloud object storage.") + + def access_table(args) -> None: storage_options = {} if args.url.startswith("s3://"): @@ -318,12 +442,71 @@ def access_table(args) -> None: print(f"Retained cache: {table.cache_bytes / 1024:8.2f} KB") +def access_external_table(args) -> None: + storage_options = None + if args.url.startswith("s3://"): + storage_options = { + "profile": args.profile, + "client_kwargs": {"endpoint_url": args.endpoint_url}, + } + + cache_options = {"cache_dir": args.cache_dir} if args.cache_dir is not None else {} + print(f"Accessing: {args.url}{f'::{TABLE_NAME}' if args.pytables else ''}") + started = time.perf_counter() + if args.pytables: + table = blosc2.RemoteCTable( + args.url, dataset=TABLE_NAME, storage_options=storage_options, **cache_options + ) + else: + table = blosc2.open( + args.url, source_format="parquet", storage_options=storage_options, **cache_options + ) + with table: + metadata_time = time.perf_counter() - started + metadata_bytes = table.traffic.nbytes + metadata_requests = table.traffic.requests + metadata = { + "type": type(table).__name__, + "rows": table.nrows, + "columns": table.col_names, + "schema": table.schema_dict(), + "attrs": dict(table.attrs), + "indexes": sorted(table._get_index_catalog()), + } + print(f"\n[Format: {'PyTables/HDF5' if args.pytables else 'Parquet'}]") + for name, value in metadata.items(): + rendered = pprint.pformat(value) if isinstance(value, (dict, list)) else value + print(f"{name:<9}: {rendered}") + + sample_start = max(0, table.nrows // 2 - 2) + sample_stop = min(sample_start + 5, table.nrows) + print(f"\nSample rows [{sample_start}:{sample_stop}]:") + print(table[sample_start:sample_stop]) + + started = time.perf_counter() + ids = table.where("(station_id == 42) & active").id[:5] + query_time = time.perf_counter() - started + print("\nQuery: (station_id == 42) & active") + print(f"first ids: {ids}") + print( + f"metadata: {metadata_time * 1000:.1f} ms, " + f"{metadata_requests} requests, {metadata_bytes / 1024:.2f} KiB" + ) + print( + f"query: {query_time * 1000:.1f} ms, " + f"{table.traffic.requests - metadata_requests} requests, " + f"{(table.traffic.nbytes - metadata_bytes) / 1024:.2f} KiB" + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "url", nargs="?", help="Local .b2z CTable path or remote URL (s3://, http://, https://)" - ) - parser.add_argument("--write", type=Path, metavar="FILE", help="Create a local .b2z CTable instead") + parser.add_argument("url", nargs="?", help="Local table path or remote URL (s3://, http://, https://)") + parser.add_argument("--write", type=Path, metavar="FILE", help="Create a local table") + formats = parser.add_mutually_exclusive_group() + formats.add_argument("--blosc2", action="store_true", help="Use Blosc2 CTable (.b2z)") + formats.add_argument("--pytables", action="store_true", help="Use PyTables/HDF5 (.h5)") + formats.add_argument("--parquet", action="store_true", help="Use Parquet (.parquet)") parser.add_argument( "--full", nargs="?", @@ -344,12 +527,25 @@ def main() -> int: if args.write is not None and args.url is not None: parser.error("URL cannot be combined with --write") if args.write is None and args.url is None: - parser.error("provide a local path or remote URL, or use --write FILE.b2z") + parser.error("provide a local path or remote URL, or use --write FILE") + if not (args.blosc2 or args.pytables or args.parquet): + source = args.write or Path(urlsplit(args.url).path or urlsplit(args.url).netloc) + format_name = {".b2z": "blosc2", ".h5": "pytables", ".parquet": "parquet"}.get(source.suffix) + if format_name is None: + parser.error("cannot infer format; use .b2z, .h5, .parquet, or a format flag") + setattr(args, format_name, True) if args.full is not None and args.write is None: parser.error("--full requires --write") + if args.full is not None and args.parquet: + parser.error("--full is not supported for Parquet") try: - write_table(args) if args.write is not None else access_table(args) + if args.write is not None: + (write_pytables if args.pytables else write_parquet if args.parquet else write_table)(args) + elif args.pytables or args.parquet: + access_external_table(args) + else: + access_table(args) except Exception as exc: print(f"Error: {exc}", file=sys.stderr) return 1 diff --git a/examples/ctable/remote_pytables.py b/examples/ctable/remote_pytables.py deleted file mode 100644 index 7df42b2e5..000000000 --- a/examples/ctable/remote_pytables.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -####################################################################### -# Copyright (c) 2019-present, Blosc Development Team -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -####################################################################### - -"""Create a PyTables table locally or access it remotely through RemoteCTable.""" - -import argparse -import pprint -import sys -import time -from pathlib import Path - -import numpy as np -import tables - -import blosc2 - -DEFAULT_PROFILE = "blosc2" -DEFAULT_ENDPOINT_URL = "https://s3.us-west-001.backblazeb2.com" -TABLE_NAME = "readings" - - -class Reading(tables.IsDescription): - id = tables.Int64Col(pos=0) - station_id = tables.Int32Col(pos=1) - temperature = tables.Float32Col(pos=2) - humidity = tables.Int16Col(pos=3) - status = tables.StringCol(8, pos=4) - active = tables.BoolCol(pos=5) - note = tables.StringCol(32, pos=6) - - -def write_table(args) -> None: - output = args.write - if output.suffix != ".h5": - raise ValueError("output must end in .h5") - if args.rows < 1 or args.batch_size < 1: - raise ValueError("--rows and --batch-size must be positive") - if output.exists() and not args.overwrite: - raise FileExistsError(f"{output} already exists; pass --overwrite to replace it") - indexed_columns = [] - if args.full is not None: - indexed_columns = ( - list(Reading.columns) if args.full == "*" else [name.strip() for name in args.full.split(",")] - ) - unknown = set(indexed_columns) - set(Reading.columns) - if not all(indexed_columns) or unknown: - raise ValueError(f"invalid --full columns: {', '.join(sorted(unknown)) or args.full!r}") - if len(indexed_columns) != len(set(indexed_columns)): - raise ValueError("--full columns must not contain duplicates") - - output.parent.mkdir(parents=True, exist_ok=True) - rng = np.random.default_rng(42) - statuses = np.array([b"ok", b"warning", b"offline", b""], dtype="S8") - notes = np.array([b"", b"clear", b"cloudy", b"rain"], dtype="S32") - - with tables.open_file(output, mode="w") as h5file: - filters = tables.Filters(complevel=5, complib="blosc2:zstd", shuffle=True) - table = h5file.create_table( - "/", - TABLE_NAME, - Reading, - title="Synthetic weather-station readings", - filters=filters, - expectedrows=args.rows, - ) - table.attrs.version = 1 - table.attrs.sampling_interval = 0.5 - table.attrs.description = "Synthetic weather-station readings" - - for start in range(0, args.rows, args.batch_size): - stop = min(start + args.batch_size, args.rows) - ids = np.arange(start, stop, dtype=np.int64) - data = np.empty(len(ids), dtype=table.dtype) - data["id"] = ids - data["station_id"] = ids % 100 - data["temperature"] = rng.normal(18, 10, len(ids)).astype(np.float32) - data["humidity"] = rng.integers(0, 101, len(ids), dtype=np.int16) - data["status"] = statuses[(ids // 7) % len(statuses)] - data["active"] = ids % 5 != 0 - data["note"] = notes[ids % len(notes)] - table.append(data) - table.flush() - - if indexed_columns: - started = time.perf_counter() - for name in indexed_columns: - getattr(table.cols, name).create_csindex(filters=filters) - print(f"Created FULL (CSI) indexes in {time.perf_counter() - started:.2f} s") - - with tables.open_file(output) as h5file: - table = h5file.root.readings - assert table.nrows == args.rows - assert all(getattr(table.cols, name).index.is_csi for name in indexed_columns) - - print(f"Created {output} ({output.stat().st_size / 1_000_000:.1f} MB, {args.rows:,} rows)") - print(f"FULL (CSI) indexes: {', '.join(indexed_columns) if indexed_columns else 'none'}") - print(f"Now upload {output} to your cloud object storage.") - - -def access_table(args) -> None: - storage_options = None - if args.url.startswith("s3://"): - storage_options = { - "profile": args.profile, - "client_kwargs": {"endpoint_url": args.endpoint_url}, - } - - print(f"Accessing: {args.url}::{TABLE_NAME}") - started = time.perf_counter() - cache_options = {"cache_dir": args.cache_dir} if args.cache_dir is not None else {} - with blosc2.RemoteCTable( - args.url, dataset=TABLE_NAME, storage_options=storage_options, **cache_options - ) as table: - metadata_time = time.perf_counter() - started - metadata_bytes = table.traffic.nbytes - metadata_requests = table.traffic.requests - metadata = { - "type": type(table).__name__, - "rows": table.nrows, - "columns": table.col_names, - "schema": table.schema_dict(), - "attrs": dict(table.attrs), - "indexes": sorted(table._get_index_catalog()), - } - print("\n[Format: PyTables/HDF5]") - for name, value in metadata.items(): - rendered = pprint.pformat(value) if isinstance(value, (dict, list)) else value - print(f"{name:<9}: {rendered}") - - sample_start = max(0, table.nrows // 2 - 2) - sample_stop = min(sample_start + 5, table.nrows) - print(f"\nSample rows [{sample_start}:{sample_stop}]:") - print(table[sample_start:sample_stop]) - - started = time.perf_counter() - ids = table.where("(station_id == 42) & active").id[:5] - query_time = time.perf_counter() - started - print("\nQuery: (station_id == 42) & active") - print(f"first ids: {ids}") - print( - f"metadata: {metadata_time * 1000:.1f} ms, " - f"{metadata_requests} requests, {metadata_bytes / 1024:.2f} KiB" - ) - print( - f"query: {query_time * 1000:.1f} ms, " - f"{table.traffic.requests - metadata_requests} requests, " - f"{(table.traffic.nbytes - metadata_bytes) / 1024:.2f} KiB" - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("url", nargs="?", help="Remote .h5 URL (s3://, http://, or https://)") - parser.add_argument("--write", type=Path, metavar="FILE", help="Create a local .h5 table instead") - parser.add_argument( - "--full", - nargs="?", - const="*", - metavar="COL1,COL2", - help="Create FULL (CSI) indexes for every column, or only the comma-separated columns", - ) - parser.add_argument( - "--cache-dir", type=Path, metavar="DIR", help="Persist remote data in DIR (default: memory)" - ) - parser.add_argument("--rows", type=int, default=1_000_000) - parser.add_argument("--batch-size", type=int, default=100_000) - parser.add_argument("--overwrite", action="store_true") - parser.add_argument("--profile", default=DEFAULT_PROFILE) - parser.add_argument("--endpoint-url", default=DEFAULT_ENDPOINT_URL) - args = parser.parse_args() - - if args.write is not None and args.url is not None: - parser.error("URL cannot be combined with --write") - if args.write is None and args.url is None: - parser.error("provide a remote URL or use --write FILE.h5") - if args.full is not None and args.write is None: - parser.error("--full requires --write") - - try: - write_table(args) if args.write is not None else access_table(args) - except Exception as exc: - print(f"Error: {exc}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plans/remote-parquet-cache-metadata.md b/plans/remote-parquet-cache-metadata.md new file mode 100644 index 000000000..abdba0227 --- /dev/null +++ b/plans/remote-parquet-cache-metadata.md @@ -0,0 +1,188 @@ +# Retain Parquet metadata for fast warm opens + +Status: implemented; ready for manual checks. Cold opens publish discovery +metadata in the generation manifest. A small local marker index finds that +generation on later opens without contacting the source. Warm opens restore +the schema before creating an Arrow reader. As with other remote table caches, +the source is assumed immutable until explicit `refresh()`. Existing caches +without the local marker index perform one source check before using this path. + +Observed in one Python process against the reported Backblaze file: cold +`open()` 1.53 s, warm `open()` 0.19 s. A reopened cached group took 0.23 s +including `open()` and transferred zero source bytes. These are single-run +measurements, not timing thresholds. + +Fresh processes using one temporary cache after the immutable-source +correction: cold `open()` 1.36 s with one HEAD and one GET (65,536 bytes read); +warm `open()` 0.029 s with no HTTP requests. A cold group read, including +`open()`, took 2.37 s with one GET; the reopened cached group took 0.051 s +with no HTTP requests. These timings exclude Python import time. + +## Objective + +Make a warm `blosc2.open(url, cache_dir=...)` reconstruct a Parquet table from +local discovery metadata. Opening the table and displaying `t.info` must avoid +downloading the footer, reading data samples, repeating schema inference, and +contacting the source. Check its version on cold open and explicit refresh. + +Keep the existing readable source directory, generation `.b2d` container, +ownership locks, converted-group cache, and portable `.b2z` exports. Store the +new discovery metadata in the generation's `embed.b2e` and a small marker +index at the cache root; no `parquet-sources/` directory is needed. + +## Existing code to reuse + +- `src/blosc2/remote_store.py`: `RemoteDiscovery._restore_manifest()` restores + discovery state before opening leaves; HDF5 reuses its parsed index. +- `src/blosc2/b2z_source.py`: `B2ZArchive` restores object information and + metadata ranges, avoiding repeated archive discovery. +- `src/blosc2/remote_store_cache.py`: `StoreDiskCache`, `SharedStoreCache`, + manifest publication, and generation ownership. +- `src/blosc2/schema_compiler.py`: `schema_to_dict()` and `schema_from_dict()` + preserve compiled schemas without repeating inference. +- `src/blosc2/remote_parquet.py`: retain its Arrow row-group adapter, source + marker validation, conversion semantics, and existing group-cache behavior. + +## Retained metadata + +Add a versioned Parquet discovery section to the existing manifest metadata. +It must contain enough information to construct the logical table without +creating a remote Arrow reader: + +- Source size and revision marker, including the Backblaze B2 file ID where + applicable. +- Arrow file metadata serialized using `FileMetaData.write_metadata_file()` + and restored using Arrow's metadata reader; do not use pickle. +- Compiled Blosc2 schema, physical/logical field mapping, and effective + conversion settings. +- Physical row-group boundaries and logical row boundaries/count. Preserve + the existing flattened-root row map when logical and physical rows differ. +- Cache/discovery compatibility version and the options identity used to + validate this state. + +Keep discovery metadata outside the converted-data eviction budget, consistent +with the other remote formats. Preserve meaningful metadata-size reporting. +Do not persist runtime credentials in the manifest. + +## Implementation sequence + +### 1. Resolve the cache before discovering the schema + +Refactor opening into local cache lookup, then either metadata restoration or +cold source validation and discovery. Cache lookup must not depend on +schema-derived values such as the inferred root name. + +Normalize the source and request options before lookup. Include projection, +row limit, reader options, effective null policy, and conversion/compression +options wherever they affect the restored table or retained groups. Preserve +the current separation of source revisions; moving revisions into generations +under a stable source directory is a separate lifecycle change. + +Define the identity compatibility path explicitly. Existing manifests lacking +discovery metadata should acquire it on the next discovery pass when their +identity remains compatible. The local marker index stores the last validated +revision and corresponding cache directory. If correcting the identity requires +a new cache directory, leave the old directory intact and document the one-time +rebuild. + +### 2. Consolidate source metadata requests + +For HTTP cold opens and explicit refresh, obtain size and supported version +headers in one metadata request using the existing fsspec session and transport +options. Capture ETag, +modification time, and Backblaze file ID without the current additional B2 HEAD. +For other filesystems, retain the appropriate `fs.info()` metadata path. + +Carry the validated size into later handle creation so HTTP fsspec does not +issue another HEAD solely to determine file size. Retain appropriate fallback +behavior for servers that cannot answer HEAD; the one-request target applies +to servers such as the reported Backblaze endpoint. + +### 3. Publish cold discovery and restore warm tables + +On a cold open, perform the existing footer read and necessary sample/row-map +discovery. Serialize the resulting state into the common manifest under its +existing lock and atomic publication mechanism. + +On a warm open, validate the discovery version and source/options identity, +restore the compiled schema and mappings, and construct `ParquetTableStorage` +directly. Avoid `CTable.from_arrow()` and remote data sampling on this path, +including for policies whose initial inference requires values. + +Validate stored schema, field mappings, counts, and row boundaries before +accepting them. Missing, incompatible, or malformed disposable discovery state +must trigger safe reconstruction; do not silently retain groups whose source +or conversion identity is uncertain. Avoid republishing unchanged manifests. + +### 4. Create the remote Arrow reader lazily + +Allow `_ParquetOwner` to hold restored metadata without an open remote handle +or `ParquetFile`. Information properties and cached-group reads must use that +state directly. + +Under the existing owner lock, create the handle and reader on the first +uncached-group request. Supply the retained `FileMetaData` through +`ParquetFile(metadata=...)` and preserve all applicable reader options. Verify +that this bypasses footer reads in the installed Arrow version. + +Make close and failed initialization safe both before and after reader +creation. Keep the existing serialization of reads through the shared handle. + +### 5. Integrate refresh, sharing, and saved artifacts + +Use the same restoration path for URL opens, generation-root opens, and saved +references. Include discovery metadata in `.save("reference.b2z")` even when +`include_cache=False`; export must not fetch unvisited groups. + +Preserve ordinary lifetime ownership and shared operation locks. Concurrent +opens must observe only fully published discovery metadata. A changed source +must not reuse old metadata or converted groups. Explicit refresh must perform +source validation and rebuild discovery when needed, preserve stale-view +semantics, and leave the old table usable if replacement preparation fails. + +Keep existing saved-reference compatibility. Older references without the new +section may perform discovery before benefiting from metadata retention. + +### 6. Document and measure the behavior + +Update `doc/guides/remote_tables.md` to explain retained discovery metadata, +the immutable-source assumption, lazy reader creation, compatibility, +and the distinction between discovery metadata and cached data groups. + +Measure the supplied Backblaze URL in fresh processes, separating Python +startup from `open()` time where useful: + +```text +https://f001.backblazeb2.com/file/blosc2/yellow_tripdata_2024-01.parquet +``` + +Report cold open, warm open with `t.info`, a cold group read, and a reopened +cached-group read. Count actual HTTP HEAD/GET requests as well as transferred +bytes: the existing table traffic counter does not include every metadata +request. Use request-count assertions for regression tests rather than brittle +wall-clock thresholds. + +## Acceptance checks + +- Warm URL open plus `t.info`: no HTTP request, no schema inference or sampling. +- Cold open: correct discovery and atomic retention, preserving existing + importer parity for nullable, nested, dictionary, and variable-length data. +- Cached-group read after reopen: no additional payload request or conversion. +- Uncached-group read: lazy reader creation uses retained metadata and fetches + only the data needed by the existing field/row-group access path. +- Projection, row limits, reader options, and conversion policies cannot reuse + incompatible restored table metadata. +- Flattened-root tables restore their logical row counts and row maps without + repeating the preparation scan. +- After explicit refresh, a source replacement, including one with the same + size but a different version marker, cannot reuse stale discovery or groups. +- Missing or corrupt discovery metadata, failed publication, and failed + refresh recover without exposing partially initialized table state. +- Shared opens, generation-root reopen, and exported references restore the + same schema and reuse retained groups. Cold exports still retain metadata. +- Closing a table before any data miss releases its cache ownership cleanly. + +Run focused Parquet regression tests and Ruff checks in the `blosc2` conda +environment. Run affected shared-cache/export tests if common code changes. +Finish with fresh-process measurements against the reported remote file and +report the observed warm-open improvement for manual checks. diff --git a/plans/remote-parquet-cache-unification.md b/plans/remote-parquet-cache-unification.md new file mode 100644 index 000000000..55699fe2b --- /dev/null +++ b/plans/remote-parquet-cache-unification.md @@ -0,0 +1,236 @@ +# Unify Parquet with the remote table cache lifecycle + +Status: cache behavior ready for manual checks. The live cache uses readable +source directories, `StoreDiskCache` generations, native group `.b2d` payloads, +and `CacheCoordinator` accounting. Generation roots reopen as Parquet tables, +and `.save("reference.b2z")` exports retained groups as a single archive. + +Implementation decision: keep the Parquet owner and storage adapter for Arrow +row groups. `RemoteDiscovery` assumes existing Blosc2 array leaves for its +format dispatch, while Parquet creates converted table segments on demand. +Forcing that class to own Parquet would add format branches across discovery, +table storage, and export. The shared cache primitives now provide the intended +layout and lifecycle behavior. The deeper `RemoteDiscovery` integration below +remains an optional architectural follow-up rather than a requirement for +manual cache checks. + +The source revision is still part of Parquet's cache identity, so `refresh()` +uses a new readable source directory after a file changes. Moving that revision +into a new generation under a stable source directory is a separate lifecycle +change; the current behavior keeps old revisions isolated and old references +detect source changes. + +## Objective + +Treat a Parquet file as a table source managed by the existing RemoteStore / +RemoteCTable machinery. Use the same source naming, generation manifests, +ownership, cache budgets, refresh, direct cache reopening, and portable exports +as other remote table formats. Keep Parquet decoding and row mapping in its +source adapter. + +The unit fetched from Parquet can remain a physical field within a row group. +Consistency does not require mapping a Parquet row group to one Blosc2 chunk. + +## Current differences + +| Concern | Existing remote store machinery | Current Parquet implementation | +| --- | --- | --- | +| Disk identity | Readable source basename plus identity hash | `parquet/<64-character hash>` | +| Live container | `.b2d` and `active_generation.json` | Independent row-group `.b2z` files and optional `.npy` row maps | +| Ownership | `StoreDiskCache`; `SharedStoreCache` for shared operations | Separate `_disk_guard` used for disk reads | +| Cache budget | `CacheCoordinator` shared by retained payloads | Private memory LRU and disk eviction by modification time | +| Refresh | Prepare/publish generation, invalidate old handles | Reopen a separate Parquet owner and change its generation counter | +| Export | Remote-store manifest and retained payloads in a `.b2z` archive | `remote_parquet` CFrame reference containing cache files as byte strings | +| Reopening live cache | Root manifest reconstructs the remote table | Individual entries open as partial ordinary CTables | + +Relevant code: `remote_parquet.py`, `remote_store.py`, `remote_store_cache.py`, +`remote_ctable.py`, `ctable_storage.py`, and `proxy.py`. + +A terminology correction: `.b2d` is a directory container. It does not imply +that every array inside it uses `contiguous=False`. The common array code has +distinct ordinary contiguous carriers and shared sparse frames. Preserve that +distinction; do not turn every Parquet payload into a sparse frame merely to +obtain a `.b2d` root. + +## Proposed layout and identity + +```text +proves/ + chicago-taxi.parquet--/ + owner.lock + active_generation.json + .b2d/ + embed.b2e + _parquet/ + row-map.b2nd # Only when flattening requires a row map + groups/ + 0/ + .b2d/ + ...native CTable payloads... + 1/ + .b2d/ + ...native CTable payloads... +``` + +Use `StoreDiskCache.path_for()` and the existing `cache_directory_name()` / +path-component helpers. Do not implement another basename sanitizer or hashing +convention. The illustrated payload subtree is private; the public entry point +is the generation root, just as for other cached remote tables. + +The identity must include the normalized source location, access-configuration +fingerprint, projection, row limit, effective null policy, and options that +change decoded values or their cache representation. Keep credentials out of +persisted descriptors. Record cache format/conversion compatibility versions +explicitly. Persist the source revision marker in the generation metadata so +refresh replaces a generation within the same logical cache directory. + +Use a versioned Parquet section of the common manifest for the compiled schema, +physical/logical field mapping, row-group boundaries, decoding options, revision +marker, row map, and published group entries. Register the root as a `ctable` +node; auxiliary group tables are implementation details, not user-visible +tables in the logical hierarchy. + +## Reuse the lifecycle, retain a small Parquet adapter + +1. Add Parquet discovery and reconstruction to `RemoteDiscovery` and the common + source-descriptor validators. Route Parquet table construction through that + owner. Preserve the existing public `blosc2.open()` options and read-only + table behavior. Keeping `RemoteParquetCTable` as a thin compatibility class is + fine; it should no longer own a separate cache lifecycle. +2. Keep fsspec/PyArrow handles, footer decoding, logical row mapping, and + `CTable.from_arrow()` conversion in `remote_parquet.py`. Preserve current + nested/null/dictionary semantics and importer parity. +3. Make `ParquetTableStorage` obtain converted groups from the common owner. + Retain its column adapter for gathering values across unequal groups. The + existing common table storage assumes B2Z members in several paths, so simply + changing the owner class is insufficient: add explicit Parquet dispatch at + those storage boundaries. +4. Store each converted group/physical-field unit as native directory-backed + CTable payloads under the generation, replacing its ZIP wrapper. This avoids + requiring global string offsets or dictionary codes before all groups have + been visited. A physical field may supply multiple logical leaves and masks; + record that mapping rather than assuming one physical field is one scalar + column. +5. Register these units with `CacheCoordinator` through one small adapter, using + the existing remote batch cache as the model. The coordinator currently + expects concrete cache methods/maps; implement that interface, not a second + coordinator or a general plugin framework. + +This is substantive lifecycle reuse, not a `.b2d` wrapper around the current +standalone `.b2z` cache directory. Directly reopening the generation must expose +the complete logical table and fetch missing groups through its source adapter. + +## Reads, publication, accounting, and sharing + +A miss converts the required physical field for its row group, including any +dependent masks, dictionaries, and nested structure. Build it under a temporary +sibling directory. Under the common ownership lock, publish the completed unit +and then its manifest entry. A crash before publication must not turn a partial +unit into a cache hit. Recover abandoned staging/unreferenced payloads through +the same generation/dirty-state machinery used for other formats. + +Retain and evict complete converted units initially, including their dependent +payloads. This keeps dictionary and offset state coherent. Cache hits touch the +common LRU; reopened caches restore accounting from validated manifest entries. +Count retained compressed data consistently with the shared coordinator and +report metadata separately. Count directory/manifest overhead separately from +the compressed-payload budget, as with other stores. + +A row group larger than the budget must still be readable. Define the retained +state using the common budget contract: serve through a temporary live unit and +release it when it cannot be retained, rather than keeping an oversized group +indefinitely. Do not claim this bounds peak Arrow conversion memory. + +Ordinary DISK access uses the existing lifetime ownership lock. Shared access +uses `SharedStoreCache` and operation-scoped locking, with the normal manifest +reload, dirty recovery, and stale-handle checks. A shared cold miss must not +publish duplicate or inconsistent units. Complete group payloads can remain +immutable after publication; sparse per-chunk writes are unnecessary for them. + +NONE and MEMORY policies retain their documented behavior through the same +owner, without creating disk artifacts. Refresh prepares compatible metadata +and a new generation before retiring the old one. Failed refresh leaves the +previous table usable. Preserve source-change detection and require explicitly +defined validation when reopening a persisted generation; do not accidentally +weaken the current Parquet revision checks while adopting shared machinery. + +## Opening and transport + +These should be equivalent ways to recover the same logical remote table: + +```python +t = blosc2.open(url, cache_dir="proves") +t = blosc2.open("proves/chicago-taxi.parquet--/.b2d") +``` + +The generation root contains enough metadata to expose the whole schema and +reuse retained groups. Unknown sizes should be reported as unavailable or +explicitly partial; information display must not fetch all groups to compute +them. Opening and reads must honor common closed/stale-handle semantics. + +Use the existing remote-store `.save()` archive writer, extending payload +enumeration for Parquet group units and the row map. The new default portable +table artifact is a genuine `.b2z` remote-reference archive: + +```python +t.save("taxi-reference.b2z") # Include currently retained data +t.save("taxi-cold.b2z", include_cache=False) +u = blosc2.open("taxi-reference.b2z") +``` + +Export must not fetch unvisited groups. Stream/copy stored payloads into the +archive rather than embedding every file as an in-memory byte string in a +CFrame metadata payload. The shared archive reader must understand Parquet +segments and serve warm data from the exported file. Uncached reads still need +the source and runtime credentials. Cache mutability on reopening should follow +the existing remote-reference rules, independently of source read-only status. + +Local Parquet sources remain supported. Keep the distinction between reopening +a local cache and transporting a remote reference: a local source path does +not become available on another machine merely because its cache was exported. + +## Compatibility and rollout + +Implement and verify the internal integration before changing the disk layout. +Do not independently land a cosmetic rename that causes an extra cache rebuild. + +- Treat old `parquet/` runtime caches as disposable. Create the new cache + on the next open; leave old directories untouched and document cleanup. No + automatic migration is needed for these runtime files. +- Preserve a read-only compatibility loader for existing version-1 + `remote_parquet` CFrame references. Seed their retained data into the common + representation after validation. New `.save()` output uses the common archive + format; the loader must not retain a second runtime cache implementation. +- Version new manifest fields and validate paths, source revision, group + identity, shape, and conversion identity before accepting a cached unit. +- Keep current import options and numerical/null semantics. Do not combine this + change with a new Parquet page reader, global dictionary scheme, or per-chunk + group conversion strategy. + +## Implementation sequence and acceptance gates + +1. **Shared source ownership:** Add the Parquet table node and descriptor + metadata, reuse ordinary/shared ownership and generations, and preserve the + existing importer parity tests, including variable-length and nested data. +2. **Native payloads and common accounting:** Replace per-group ZIP files with + native group directories, integrate `CacheCoordinator`, and test publication, + eviction, reopened cache hits, and shared concurrent misses. +3. **Reopen/export integration:** Open the generation root as a full + `RemoteCTable`, export/import standard `.b2z` references, and support old saved + references through the compatibility loader. +4. **Remove duplicate paths and document:** Remove Parquet-specific path naming, + disk locking, LRU/eviction, lifecycle, and new-reference CFrame serialization. + Document the common layout and remaining Parquet fetch granularity. + +Completion requires tests covering unequal row groups, a read of another column +and another group, repeated warm reads with no payload refetch, persisted reopen, +complete logical schema at the generation root, refresh/stale views, aborted +publication, oversized units, and two processes sharing a cold miss. Export to +a different directory/process must preserve warm reads and allow cold misses to +fetch the source; old-reference loading must retain its source checks. Run the +existing remote-store/CTable cache and export tests alongside Parquet parity. + +Compare cold open, one narrow cold read, warm read, reopen, and export on a +multi-group mixed-type fixture. Record source traffic, retained bytes, and file +count. Changes must preserve column/group isolation and avoid repeated +conversion of retained data; no full-file import is an acceptable fallback. diff --git a/plans/remote-parquet.md b/plans/remote-parquet.md new file mode 100644 index 000000000..48fed2f0b --- /dev/null +++ b/plans/remote-parquet.md @@ -0,0 +1,580 @@ +# Remote Parquet tables: implementation plan + +Status: implementation in progress; final parity gate remains open. +Baseline: Python-Blosc2 working tree inspected on 2026-09-25. + +## Goal and completion contract + +Support read-only remote Parquet tables through the existing fsspec/remote-table +API, with at least the data types, conversions, selection options, and logical +results currently supported by `CTable.from_parquet()`. Implement this in small, +independently tested iterations. A numeric-only prototype is an intermediate +milestone, not completion of this goal. + +For a supported single Parquet file and equivalent conversion options, compare +the remote table with `CTable.from_parquet()` on the same file. Require matching +logical schema, column order, row count, values, nulls, nested row reconstruction, +query results, and Arrow export semantics. Match existing documented conversion +and rejection behavior; this does not promise lossless handling of types or +metadata that the current importer already transforms or rejects. + +Ordinary opening should read metadata alone for mask-backed nulls; in-band null +policies may still require an inference sample. Row reads should fetch the necessary physical column chunks +and row groups, without importing the entire table. Some existing conversions +need more preparation, particularly unnamed-root flattening and dictionary +normalization. Account for and document those reads rather than claiming all +files can open from the footer alone. + +Preserve `CTable.from_parquet()` as the eager, batched import API. Destination +creation options apply when materializing a remote table; a remote source remains +read-only. No silent eager-import fallback for unsupported lazy cases. + +## Existing implementation to reuse + +- `src/blosc2/ctable.py`: `from_parquet()`, `from_arrow()`, Arrow schema compilation, + null policy, nested flattening, and batch conversion. The importer passes its + input directly to `pyarrow.parquet.ParquetFile` and forwards reader kwargs. +- `src/blosc2/remote_ctable.py`: read-only table API, lifetime management, tuning, + persistent carriers, and materialization behavior. +- `src/blosc2/remote_store.py`: format discovery, shared source ownership, + generation checks, filesystem access, and table discovery. +- `src/blosc2/ctable_storage.py`: `RemoteTableStorage`, synthetic all-valid rows, + fixed and variable column access, and the existing HDF5-specific integration. +- `src/blosc2/ctable_remote_read.py`: projected/gathered table reads and bounded + parallel work. Audit its assumptions about Blosc2 arrays and batch backends. +- `src/blosc2/remote_batch.py`, `remote_store_cache.py`, and + `remote_source_cache.py`: batch access, cache ownership/accounting, and source + metadata reuse. Reuse compatible pieces; a Parquet column chunk is not a B2Z + member and cannot simply inherit its addressing rules. +- `src/blosc2/schunk.py`, `core.py`, and `ref.py`: opening, URL parsing, source + descriptors, and reopening. Trace all format allowlists and serialized-source + validators before adding the new format. +- `pyproject.toml`: `blosc2[parquet]` already supplies PyArrow. fsspec and protocol + drivers remain the transport dependencies. + +A prior read-only probe verified `CTable.from_parquet(file_handle)` and +`CTable.from_parquet(path, filesystem=fs)` with fsspec's memory filesystem. This +establishes transport compatibility for eager import, not lazy access or HTTP/S3 +performance. The CLI additionally assumes local `Path` inputs; changing it is +not required for the lazy backend. + +## Compatibility inventory + +Turn this inventory into executable cases in iteration 1. Use the actual eager +result as the reference, including edge cases where physical Arrow types change +after a Parquet round trip. + +| Capability | Required outcome | Main implementation concern | +| --- | --- | --- | +| Signed/unsigned integers, float32/64, Boolean | Same values, dtypes, operations | Row-group to logical-row mapping; boundaries and NaNs | +| Timestamps | Same units, timezone metadata, nulls, comparisons | Share importer conversion rules | +| UTF-8 and binary, including supported large/fixed variants | Same logical strings/bytes and empty values | Variable payload sizes and offset addressing | +| `string_max_length` scalar/mapping | Same fixed-width selection and overflow errors | Freeze inferred schema before out-of-order reads | +| Nullable scalars | Same mask/sentinel behavior and null-policy precedence | Keep valid sentinel-like values distinct in mask mode | +| Dictionary strings | Same supported categorical semantics | Dictionaries/codes can differ between row groups | +| Fixed-size lists / shaped cells | Same mapped CTable shape and values | Field metadata, row-level and child null semantics | +| Lists and large lists | Same list values and supported children | Variable element counts; null versus empty list | +| Nested lists and list-of-struct | Same nested values | Both `msgpack` and `arrow` list serializers | +| Top-level structs | Same flattened field paths and reconstructed rows | Parent validity, escaped names, leaf projection | +| Unnamed fields | Same collision-safe root renaming | Store reversible physical/logical name mapping | +| Single unnamed list-of-struct root | Same flattened element rows by default | Footer row counts describe outer rows, not output rows | +| `separate_nested_cols=False` | Same non-separated representation | Keep physical outer-row semantics | +| `columns` | Same accepted names, order, and errors | Import-time selection versus later leaf projection | +| `max_rows` | Same prefix, including zero | Applies after unnamed-root flattening | +| Empty data | Same schema and empty results | No first batch available for inference | +| Unsupported types | Same deliberate rejection | No new implicit object fallback | +| Reader kwargs | Equivalent PyArrow decoding behavior | Separate transport, conversion, and reader options | +| Batch/compression/validation options | Same logical conversion behavior | Apply storage tuning to cache/materialization appropriately | +| Schema and nested metadata | Same metadata semantics as eager import | Do not accidentally promise extra CLI-only metadata handling | + +The authoritative starting tests are `tests/ctable/test_parquet_interop.py`, +`test_arrow_interop.py`, and `test_null_mask_arrow.py`. Arrow-only cases become +requirements only where the same case is accepted by `from_parquet()` after +writing/reading Parquet. In particular, `from_arrow(object_fallback=True)` is not +a capability of `from_parquet()`. + +## Proposed public API + +Use `blosc2.open(url, source_format="parquet", ...)` for a single table at the +file root, returning `RemoteCTable`. Recognize `.parquet` automatically, including +URLs with query parameters; explicit format handles extensionless URLs. Preserve +the existing `lazy` convention, and document both lazy and explicit eager paths. +Allow direct `RemoteCTable` construction using the same source options. + +Illustrative target API, subject to the iteration-1 keyword audit: + +```python +table = blosc2.open( + "s3://bucket/data.parquet", + source_format="parquet", + lazy=True, + storage_options={"anon": True}, + columns=["temperature", "station"], + cache_dir="cache", + parquet_options={"read_dictionary": ["station"]}, +) +``` + +Keep `storage_options` for fsspec credentials/configuration. Put PyArrow reader +kwargs in a dedicated `parquet_options` mapping to avoid collisions with +`blosc2.open()` and conversion parameters. Expose conversion parameters matching +the importer where relevant: `columns`, `max_rows`, `string_max_length`, +`null_storage`, `auto_null_sentinels`, `separate_nested_cols`, `list_serializer`, +and batch settings. Resolve the exact naming once, in iteration 1. + +`urlpath`/`mode` used as importer destination options belong to materialization, +not source mutation. Keep compression/decompression and validation settings +usable for converted caches and materialized output. Do not silently ignore +options that lack a remote meaning. Reader settings requiring local files, such +as memory mapping, need explicit documented behavior on remote handles. + +Credentials, live filesystems, decryption objects, and other runtime-only reader +objects must not be serialized into portable carriers. Direct reads may use +them; reopening must accept replacement runtime options or fail clearly. Resolve +ambient null policy when opening and record the effective conversion policy so +later reads cannot change with a different context. + +## Internal approach + +Add a focused Parquet source module rather than a new general storage framework. +One owner holds the filesystem handle, `ParquetFile`, footer/schema, row-group +boundaries, effective conversion options, and shared cache state. Expose the file +as a table node through the existing remote ownership model. + +Map requested logical rows to row groups using cumulative counts. Group gathers +by physical group, read the union of requested physical fields once, convert +through shared Arrow/CTable rules, and restore caller order and duplicates. +Separate logical leaf names from Parquet physical paths. Projected nested fields +may need parent validity or other structural data; request that explicitly. + +Keep Parquet knowledge at this source/storage boundary. Existing CTable queries +and exporters should consume ordinary column semantics. Use the existing column +and batch interfaces where they fit, with the smallest necessary adapter where +they do not. Prototype both scalar and variable-length access before fixing the +internal interface: global UTF-8 offsets and dictionary vocabularies are not +available by pretending each Parquet group is an independent whole table. + +Prefer converted Blosc2 cache units keyed by physical row group and logical +column/conversion identity. Keep related validity and dictionary dependencies +consistent. Transient Arrow buffers must be bounded and separately accounted +for. Avoid repeatedly decoding one large group for adjacent small Blosc2 chunks. +Do not build a second persistent raw-byte cache unless measurement shows a need. + +## Iterations and acceptance gates + +Each iteration leaves a usable, tested increment and updates this document with +completed scope, remaining restrictions, and measured behavior. Complete gates in +order; failures in schema or row semantics block performance work. + +### 1. Freeze the compatibility contract and prove the storage seam + +- Inventory the importer tests and produce a compact parameterized parity matrix. + Include schema, null policy, metadata, reader kwargs, and invalid inputs. +- Trace source opening, remote storage, column reads, query evaluation, exports, + serialization, and close/refresh paths. Record format-specific assumptions. +- Prototype one numeric column and one UTF-8/list column through the proposed + storage seam. Determine whether to extend `RemoteTableStorage` directly or add + a small Parquet-specific storage implementation sharing its lifecycle. +- Extract only shared importer normalization/conversion code that the prototype + actually needs. Preserve eager behavior and avoid duplicating type mappings. +- Resolve API keyword routing, capability errors during intermediate stages, and + metadata-versus-sample requirements. Keep experimental APIs private until stable. + +Gate: representative remote reads match eager import, existing import tests pass, +and the chosen storage seam supports both fixed and variable payloads without +whole-file import. Record any data-dependent schema inference discovered. + +### 2. Single-file lazy access for scalar columns + +- Open local and fsspec sources, obtain footer/schema, and expose a table node. +- Implement numeric, Boolean, timestamp, and nullable scalar columns, synthetic + all-valid rows, projection, ordinary `max_rows`, and empty tables. +- Implement row indexing/slicing/gathers to the extent supported by the existing + CTable API, including negative indices, reordered/duplicate positions, empty + selections, bounds errors, and reads spanning unequal row groups. +- Integrate normal CTable scan queries and basic Arrow/materialized output. +- Own/close file handles correctly; reject writes and unsupported types clearly. + Add initial shared in-memory cache and source generation checks. + +Gate: scalar parity and scan results pass. Instrumented reads prove a narrow +selection does not scan unrelated row groups or columns, allowing documented +footer/prefetch overhead. Close and failure paths leak no owned handles. + +### 3. Strings, binary, and dictionary columns + +- Implement UTF-8, supported binary variants, fixed-width overrides, and nulls. + Preserve empty string/bytes separately from missing values. +- Reuse width inference exactly where the importer relies on it; inference must + be deterministic, independent of which row the user accesses first. +- Supply lazy variable-length column access without building global payload + offsets by scanning all strings at open. +- Handle dictionary evolution and dictionary order differences across groups. + Reuse the importer's dictionary semantics. If the CTable representation needs + a global vocabulary, perform an explicit bounded-memory preparation pass over + the affected column, cache its result, and account for its cost. Never expose + group-local codes as if they were global or mutate published code meanings. + +Gate: round trips, filtering, null masks, multibyte strings, binary zero bytes, +overflow errors, dictionaries across multiple groups, and out-of-order reads +match eager import. Warm reads reuse converted data within the cache budget. + +### 4. Structs, shaped cells, and ordinary nested lists + +- Implement top-level struct flattening, field-name escaping, reconstruction, + and logical-leaf to physical-field projection using shared importer helpers. +- Implement fixed-size lists/shaped cells and all list/nested combinations + accepted by the importer, with both serializers and batch-setting variants. +- Preserve parent and child validity as the importer does. Cover null structs, + lists containing nulls, empty lists, null lists, and nested timestamps. +- Support unnamed-field renaming and `separate_nested_cols=False` behavior. +- Verify batch boundaries independent of row-group boundaries and large cells + exceeding nominal buffer targets. Document indivisible allocation limits. + +Gate: the ordinary-row portion of the complete parity matrix passes, including +nested queries, gathered reads, Arrow export, and materialization/reopen. + +### 5. Flattened unnamed-root row semantics + +This is required for the goal, not deferred nested-data polish. + +- Reuse detection and flattening for a single unnamed list/large-list of struct. + Distinguish physical outer rows from logical element rows everywhere. +- Build cumulative logical element counts per physical row group. PyArrow may + have to decode nested data to recover the required lengths; do not assume + footer counts or a cheap offsets-only API exist. +- Start with a bounded-memory preparation scan of the required nested data when + exact `len()` and random indexing need a complete map. Persist this map by + source generation and conversion policy. Reuse decoded groups when practical. +- If `max_rows` bounds the logical prefix, stop preparation once that prefix is + known, or at EOF when the source contains fewer elements. Zero must yield the + correct empty schema without a payload scan unless inference truly requires it. +- Match null/empty outer-list handling, field metadata, reconstructed rows, and + the different semantics when separation is disabled. + +Gate: logical row counts and values match eager import across group boundaries, +null/empty outer rows, reordered reads, and limits. Reopening a valid cached map +avoids repeating preparation. Preparation traffic is reported separately. + +### 6. Persistent cache, source identity, and portable reopening + +- Extend existing cache/carrier descriptors and format validators for Parquet. + Include format version, normalized source identity/generation, column mapping, + selection, effective null/conversion policy, reader semantics, and row-map + identity. Include storage representation settings where they affect cache data. +- Follow the existing immutable-source/explicit-refresh contract. Refresh + invalidates dependent schemas, row maps, dictionaries, and payloads together; + old views/columns must fail as stale rather than mix generations. +- Support the existing applicable cache policies, aggregate byte budget, eviction, + disk reuse, and shared sparse-cache entry points. Eviction must not leave data + without its required validity/dictionary dependencies. +- Publish converted units and preparation metadata atomically, using existing + locks/publication mechanisms. Interrupted or corrupt caches rebuild safely. +- Implement carrier save/reopen, materialization to ordinary writable CTable, + runtime filesystem/credential reinjection, and clear failures for nonportable + reader options. Keep source credentials out of serialized metadata and logs. +- Exercise concurrent reads and shared ownership; serialize access to a shared + seekable Python handle unless safe concurrent access is established. + +Gate: cold/warm/reopened results agree, retained-cache reads avoid extra payload +fetches, limits/eviction work, source refresh cannot mix generations, and failed +publication cannot expose partial tables. Test memory and disk policies as well +as shared-cache behavior supported by the final API. + +### 7. Complete the parity gate and public integration + +- Finish `blosc2.open`, direct constructor, extensionless URL, format detection, + source descriptor, and materialization option routing. Ensure `.parquet` + detection does not disturb Zarr/HDF5/B2Z opening or signed URL handling. +- Run the full compatibility matrix against eager imports, with each current + capability assigned a passing case. Cover reader kwargs such as dictionary + decoding and timestamp coercion; validate transport-incompatible combinations. +- Exercise normal column access, projections, views, filters/reductions where + supported, Arrow streaming, Parquet export, and local persistence/reopen. + Queries may scan; predicate pushdown is not needed for semantic parity. +- Add a counted HTTP range-server test to validate real seek/range behavior and + a small optional S3-compatible network smoke test. Document servers without + range support: either require explicit download/import or fail clearly. +- Measure open/preparation time, requests/bytes, peak temporary memory, cold and + warm reads, and conversion/cache cost for representative scalar, wide, + variable-length, dictionary, and unnamed-root files. +- Add user documentation and runnable examples explaining lazy reads, projected + columns, preparation costs, cache behavior, materialization, and reader options. + +Gate: all baseline capabilities are supported with documented remote costs; +existing import and remote-table regression tests pass. Report any newly found +baseline bugs separately and agree their resolution before calling parity done. + +## Validation strategy + +Use the required `blosc2` conda environment. Keep new coverage in a focused +`tests/ctable/test_remote_parquet.py`, reusing existing fixtures/helpers where +practical. Parameterize meaningful compatibility cases rather than copying the +entire importer suite. Add format-routing/cache tests to their existing modules +only where those shared contracts change. + +For each fixture, write Parquet locally, import it eagerly, expose the same bytes +through fsspec, and compare remote behavior. Compare null-aware values and logical +schema explicitly; plain equality is insufficient for NaN, nested nulls, and +dictionary encoding. Test nonuniform row groups and mismatched Arrow/Blosc2 batch +sizes to expose accidental alignment assumptions. + +Instrument both transport reads and decoder calls: byte caching can hide repeated +decoding, and decoder counters can hide excessive network prefetch. Use structural +assertions about touched groups/fields and bounded buffers, not brittle exact +request counts tied to one PyArrow release. Large indivisible values and decoded +Arrow allocation overhead must be reported rather than hidden behind a compressed +cache budget. + +Run focused tests after each iteration. At final integration, run the relevant +Parquet/Arrow/null-mask suites plus remote CTable/store/array regressions and the +repository's required checks. Network tests should remain explicitly marked and +must not be necessary for deterministic local correctness coverage. + +## Scope after the goal + +The following can be separate iterations after single-file importer parity: + +- Partitioned/multi-file datasets, directory discovery, and schema unification. +- Parquet statistics/page-index predicate pushdown and native index generation. +- Remote Parquet writes, updates, or transactional mutation. +- Caterva2 server/UI discovery and serving. This needs a separate cross-repository + pass, including format allowlists and source validation; Python-Blosc2 support + alone does not establish Caterva2 support. +- CLI URL convenience, specialized prefetch, and page-level decoding optimization. + +These exclusions do not remove any single-file data capability currently accepted +by `CTable.from_parquet()`. Additional unsupported types or implicit object +fallbacks are outside the compatibility target unless requested separately. + +## Progress ledger + +Prototype status (2026-09-25): `src/blosc2/remote_parquet.py` provides a read-only +row-group adapter for local and fsspec single files. `blosc2.open()` and direct +`RemoteCTable` construction recognize Parquet, with explicit eager opening via +`lazy=False`. It uses `CTable.from_arrow()` for schema compilation and for +on-demand conversion of physical fields, so the tested scalar, UTF-8, list, +struct, fixed-size-list, dictionary, and flattened unnamed-root results match +`CTable.from_parquet()`. Explicit refresh invalidates old views. `cache_dir` +retains converted row groups and flattened-root row maps across opens, with a +byte budget and corrupt-entry rebuilding. File locking permits concurrent +processes to reuse a disk cache. CFrame-backed references can retain converted +groups and logical row maps, reopen through `blosc2.open()`, and accept +replacement runtime options through `RemoteCTable.open_reference()`. A local HTTP range-server test confirms +that a narrow cold read transfers less than the file and a warm read transfers +zero bytes. After the request optimizations, the focused Arrow/Parquet, +remote-table, and open regressions pass (458 cases). The full default suite +passed (10,633 passed, 36 skipped) before these optimizations. This is an +intermediate increment. + +Current limits: in-band null policies may sample the first row group; flattened +unnamed roots scan one leaf per group for exact logical counts. Converted +groups are retained in an in-memory LRU or source-identified disk files. +Credential replacement is covered with a runtime fsspec option, but an actual +authenticated service is not yet tested. More importer edge cases, reader-option +routing, memory measurements, and optional S3 smoke coverage remain. +The first attempt to add nullable fixed-size-list parity exposed a baseline +PyArrow `iter_batches()` error when reading that Parquet file: "Expected all lists +to be of size=2 but index 2 had size=0." The eager importer fails before remote +comparison; the valid fixed-size-list variant passes. Nullable fixed-size lists +remain outside remote parity while the eager importer cannot read them. +These missing pieces keep the final gate open. + +The in-memory benchmark `bench/remote_parquet_traffic.py` counts file-handle +read calls and bytes returned, which approximate requests/transfer for this +transport but do not model HTTP or S3 buffering. With 10,000 rows in ten row +groups, the pre-optimization baseline measured: + +| Fixture | Parquet bytes | Open reads / bytes | Cold final-row reads / bytes | Warm reads / bytes | +| --- | ---: | ---: | ---: | ---: | +| 2-column scalar | 109,920 | 1 / 65,536 | 1 / 5,364 | 0 / 0 | +| 20-column wide | 1,096,185 | 1 / 65,536 | 1 / 5,364 | 0 / 0 | +| UTF-8 strings | 64,961 | 1 / 64,961 | 1 / 6,352 | 0 / 0 | +| Dictionary strings | 8,231 | 2 / 15,241 | 1 / 701 | 0 / 0 | +| Flattened unnamed root | 55,530 | 12 / 162,866 | 1 / 5,368 | 0 / 0 | + +Open bytes may exceed file size because the in-memory handle returns overlapping +footer and sampled-group reads. Scalar and UTF-8 opens use footer-only schema +compilation. Dictionary opening samples one row group. The unnamed-root open +includes its logical-row preparation scan. + +The reproducible script now also times each stage and reports converted-cache +bytes and `tracemalloc`'s peak Python allocation. On this host, traced peaks +were 494,333 B (scalar), 1,371,630 B (wide), 197,234 B (strings), 314,839 B +(dictionary), and 155,616 B (unnamed root). These exclude Arrow's native buffers +and are not a bound on process RSS or indivisible row-group allocation. + +The counted localhost HTTP range server, serving a 119,757-byte two-column +file with ten row groups, returned 1 request / 65,536 bytes on open, +1 request / 5,364 bytes for a cold `x[9500]`, and 0 / 0 for the warm repeat +with `storage_options={"block_size": 4096, "cache_type": "none"}`. These are +server-side GET response totals, unlike the in-memory file-handle figures above; +metadata HEAD requests are separate. A second localhost test verifies that a +server ignoring ranges fails with fsspec's range-request error. An optional +`network`-marked S3 smoke test uses `BLOSC2_REMOTE_PARQUET_S3_URL`. + +### Chicago taxi files (local range-read baseline) + +`bench/remote_parquet_files.py` measures the same cold and warm access stages +against existing files without copying them. It compares the last-row value +with PyArrow's last row group. These are **local file-handle read calls and bytes +returned**, not HTTP or S3 wire requests. Times include row-group conversion. +The exact measurements are saved in `bench/remote_parquet_chicago_results.json`. +The measurements below use the default in-memory cache and `trip.sec` unless +another column is named. + +| File / column | File bytes | Logical rows | Open calls / bytes / s | Cold calls / bytes / s | Warm calls / bytes / s | Arrow value | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 581,387,271 | 24,314,322 | 2 / 32,679,571 / 1.242 | 1 / 331,590 / 17.690 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 686,857,703 | 24,314,322 | 2 / 33,057,447 / 1.087 | 1 / 352,939 / 17.802 | 0 / 0 / 0.066 | match | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 687,450,036 | 24,314,322 | 2 / 33,068,884 / 1.099 | 1 / 352,939 / 17.988 | 0 / 0 / 0.067 | match | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 581,387,271 | 24,314,322 | 2 / 32,679,571 / 1.339 | 1 / 1,831 / 0.862 | 0 / 0 / 0.069 | match | +| `chicago-taxi.parquet` / `trip.sec` | 640,173,859 | 24,314,322 | 27 / 671,548,047 / 3.932 | 1 / 6,660,483 / 1.077 | 0 / 0 / 0.067 | match | + +The original `chicago-taxi.parquet` is a single unnamed nested list-of-struct +field. Its footer has 7,728 outer rows in 25 groups, and the lazy flattened +view opens with 24,314,322 logical rows. Opening reads 27 times and returns +671,548,047 bytes in 3.932 seconds because it scans every group to build the +logical-row map. An initial attempt exposed an expensive schema probe that +converted the entire first group; the probe now uses one outer row. The +synthetic parity suite still passes after this change. Its last `trip.sec` +value matches the nested Arrow leaf in the final physical group. + +### Chicago taxi files over localhost HTTP + +The same five measurements were repeated with +`bench/remote_parquet_files.py --http`. Its localhost server honors byte ranges +and counts completed GET responses and response-body bytes at the server. The +fsspec client uses `block_size=4096` and `cache_type="none"`; converted row groups +remain in the default in-memory cache. The exact results are saved in +`bench/remote_parquet_chicago_http_results.json`. Each open also made two HEAD +requests; cold and warm reads made no HEAD requests. Bytes exclude HTTP headers +and transport overhead. Times include local loopback transfer and conversion. + +| File / column | Open GETs / bytes / s | Cold GETs / bytes / s | Warm GETs / bytes / s | Arrow value | +| --- | ---: | ---: | ---: | --- | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 2 / 32,679,571 / 1.253 | 1 / 331,590 / 17.688 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 2 / 33,057,447 / 1.084 | 1 / 352,939 / 17.885 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 2 / 33,068,884 / 1.122 | 1 / 352,939 / 18.131 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 2 / 32,679,571 / 1.308 | 1 / 1,831 / 0.813 | 0 / 0 / 0.068 | match | +| `chicago-taxi.parquet` / `trip.sec` | 27 / 671,548,047 / 3.909 | 1 / 6,660,483 / 1.051 | 0 / 0 / 0.067 | match | + +The HTTP GET bytes matched the local file-handle bytes for each phase. The +unnamed nested root remains expensive to open because the exact flattened row +count requires reading all groups. For the flat numeric files, the roughly +18-second cold access is dominated by converting a whole row group, despite a +single sub-megabyte GET for `trip.sec`. + +For `chicago-taxi-flat-f32.parquet`, the final group has 259,680 rows and its +`trip.sec` chunk is 352,939 compressed bytes. Reading and decoding that chunk +directly with PyArrow took about 0.001 seconds from the local file. A `cProfile` +trace of the cold remote access showed 259,680 calls to `_ChunkAlignedWriter._write` and +259,712 calls to `NDArray.__setitem__` during `CTable.from_arrow()` conversion. Profiling raised +the total to 32.4 seconds versus 17.9 seconds uninstrumented. Those calls came +from one-row Blosc2 chunks created without a capacity hint. Passing the known +row-group length fixes this bottleneck, as measured below. + +### Request and transfer optimization (2026-09-25) + +The earlier tables are the baseline. The updated localhost results are saved in +`bench/remote_parquet_chicago_http_optimized.jsonl`. Default mask-backed schema +inference now uses the Arrow footer schema without reading a data sample, and +ordinary in-memory opens defer the source-identity lookup until a reference is +saved. This changes flat-file opening from two GETs, about 33 MB and two HEADs +to one 65,536-byte GET and one HEAD. Cold and warm row reads are unchanged. + +For an unnamed `list` root, the exact logical row count still requires +examining every row group. The scanner now projects the smallest compressed +Parquet leaf in each group; that leaf carries the outer list's repetition levels. +The Chicago nested file's open went from 27 GETs / 671,548,047 bytes / 3.909 s +to 26 GETs / 333,139 bytes / 0.364 s. A disk-cached row map reduces a reopened +nested file to one footer GET / 65,536 bytes. The HTTP source marker now accepts +fsspec's `Last-Modified` key, allowing this cache to work with HTTP servers that +provide that header. + +| File / column | Open GETs / bytes / s | Cold GETs / bytes / s | Warm GETs / bytes / s | Arrow value | +| --- | ---: | ---: | ---: | --- | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 1 / 65,536 / 0.188 | 1 / 331,590 / 17.571 | 0 / 0 / 0.068 | match | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 1 / 65,536 / 0.022 | 1 / 352,939 / 17.638 | 0 / 0 / 0.068 | match | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 1 / 65,536 / 0.016 | 1 / 352,939 / 17.862 | 0 / 0 / 0.069 | match | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 1 / 65,536 / 0.189 | 1 / 1,831 / 0.819 | 0 / 0 / 0.068 | match | +| `chicago-taxi.parquet` / `trip.sec` | 26 / 333,139 / 0.364 | 1 / 6,660,483 / 1.083 | 0 / 0 / 0.068 | match | + +The [fsspec caching strategies](https://filesystem-spec.readthedocs.io/en/latest/features.html) +and [h5py chunk/page buffering](https://docs.h5py.org/en/latest/high/file.html) +motivated a read-ahead check. On the old full-column nested scan, 16, 64 and +128 MiB fsspec read-ahead blocks used 25, 9 and 6 GETs respectively versus 26 +without buffering, but all transferred about 640 MB and took about 3.8–3.9 s. +The leaf projection removes almost all of that transfer, while large read-ahead +would fetch unrelated fields and groups, so it is left as an explicit fsspec +`storage_options` choice rather than a default. The converted-row-group cache +continues to eliminate requests for repeated values. + +The synthetic benchmark was rerun and saved in +`bench/remote_parquet_synthetic_optimized.txt`. Dictionary opening fell from +2 reads / 15,241 bytes to 1 read / 8,231 bytes. The flattened-root open fell +from 12 reads / 162,866 bytes to 11 reads / 109,198 bytes. Scalar, wide, and +UTF-8 open counts stayed at one read. + +### Row-group conversion capacity (2026-09-25) + +The remote reader now passes the known logical row-group length as +`capacity_hint` to `CTable.from_arrow()`. Without it, Arrow import sized the +flat column's initial Blosc2 chunks for one row, causing many small writes. +The localhost rerun is saved in +`bench/remote_parquet_chicago_http_capacity_hint.jsonl`; values still match +PyArrow. Request counts and transferred bytes are unchanged from the preceding +HTTP table. + +| File / column | Cold seconds before | Cold seconds after | Speedup | +| --- | ---: | ---: | ---: | +| `chicago-taxi-flat-f32-cl20.parquet` / `trip.sec` | 17.571 | 0.080 | 220× | +| `chicago-taxi-flat-f32.parquet` / `trip.sec` | 17.638 | 0.083 | 213× | +| `chicago-taxi-flat-f64.parquet` / `trip.sec` | 17.862 | 0.085 | 210× | +| `chicago-taxi-flat-f32-cl20.parquet` / `company` | 0.819 | 0.089 | 9.2× | +| `chicago-taxi.parquet` / `trip.sec` | 1.083 | 1.087 | 1.0× | + +Times are single local runs, including loopback transfer, conversion, and +value retrieval. The nested case already received a large logical capacity +through the Arrow importer's unnamed-root fallback. The flat converted-group +cache footprint also fell from about 34.7 MB to 356 KB for `trip.sec`. + +### Nested leaf projection (2026-09-25) + +The remote reader now maps a logical nested leaf to an unambiguous Parquet +column path using footer metadata. It falls back to the containing field when +the mapping is ambiguous. The converted cache is keyed by the projected path, +so sibling leaves do not force each other into the same conversion. Results are +saved in `bench/remote_parquet_chicago_http_leaf_projection.jsonl`. + +For `chicago-taxi.parquet / trip.sec`, opening still reads 26 GETs / 333,139 +bytes for the exact logical row map. The cold final-row read fell from one GET / +6,660,483 bytes / 1.087 s to one GET / 363,880 bytes / 0.073 s. The warm repeat +still makes no GET. Converted cache size fell from 12.6 MB to 356 KB. The other +four Chicago HTTP cases kept their request and transfer counts and had cold +times of 0.081–0.094 s. All measured final values match PyArrow. A local +comparison of the last nested row group also matched all 14 logical columns +against a full-field import, including the list-valued `trip.path` column. + +- [ ] 1. Compatibility inventory, API contract, and storage seam +- [ ] 2. Scalar lazy access +- [ ] 3. Strings, binary, and dictionaries +- [ ] 4. Structs, shaped cells, and nested lists +- [ ] 5. Unnamed-root flattening and logical row map +- [ ] 6. Persistent caching and portable lifecycle +- [ ] 7. Complete parity, transport verification, and documentation + +Do not estimate completion from the numeric prototype alone. The largest +uncertainties are schema inference shared with the importer, global dictionary +semantics, variable-length column interfaces, and the preparation required for +flattened root rows. Re-estimate remaining work after iterations 1 and 5. + +## Library references + +- [PyArrow ParquetFile](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html): + metadata, selected-column row-group reads, and batch iteration. +- [Arrow Parquet documentation](https://arrow.apache.org/docs/python/parquet.html): + format layout and filesystem integration. + +Use PyArrow for decoding and fsspec for transport. Verify behavior against the +installed/supported versions during implementation, especially nested projection, +reader options, prefetch, and Python-file concurrency. diff --git a/pyproject.toml b/pyproject.toml index 13a2ce117..2ffc2c967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,8 @@ test = [ # feature silently skips in CI. memory:// covers the protocol-generic paths, # and is also the configuration most users installing [fsspec] are in. "fsspec; platform_machine != 'wasm32'", + # Exercise Parquet import and lazy remote-table tests in native CI jobs. + "pyarrow; platform_machine != 'wasm32'", # Exercise the optional remote array sources instead of silently skipping # tests/test_zarr_source.py and tests/test_hdf5_source.py. "zarr>=3.0.9; platform_machine != 'wasm32'", diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 67b240def..8d686c97d 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -769,6 +769,8 @@ def parse_container_url( return urlpath, dataset, "hdf5" if any(part.endswith(".zarr") for part in parts): return urlpath, dataset, "zarr" + if parsed.path.lower().endswith(".parquet"): + return urlpath, dataset, "parquet" return urlpath, dataset, None @@ -811,8 +813,11 @@ def cache_path_component(value: str) -> str: def cache_directory_name(urlpath: str, identity: bytes) -> str: """A recognizable source basename and a 48-bit cache identity.""" - parsed = urllib.parse.urlsplit(urlpath) - name = pathlib.PurePosixPath(parsed.path.rstrip("/")).name or parsed.hostname or "remote" + if pathlib.PureWindowsPath(urlpath).drive and not is_fsspec_url(urlpath): + name = pathlib.PureWindowsPath(urlpath).name + else: + parsed = urllib.parse.urlsplit(urlpath) + name = pathlib.PurePosixPath(parsed.path.rstrip("/")).name or parsed.hostname or "remote" name = cache_path_component(urllib.parse.unquote(name)) return name + "--" + hashlib.sha256(identity).hexdigest()[:12] diff --git a/src/blosc2/ctable_remote_read.py b/src/blosc2/ctable_remote_read.py index 493366436..ada14ca69 100644 --- a/src/blosc2/ctable_remote_read.py +++ b/src/blosc2/ctable_remote_read.py @@ -345,6 +345,30 @@ def column_values(table, names, positions, *, null_masks=None): # noqa: C901 ) storage = table._remote_read_storage() + if storage._owner.format == "parquet": + with storage._owner.lock: + storage._check_open() + + def reader(name): + if name not in table.col_names: + return table._fetch_col_at_positions_uncached(name, positions) + values = yield from table._cols[name].reads(positions) + mask = table._null_mask(name) + if mask is not None: + missing = ~(yield from mask.reads(positions)) + if null_masks is not None: + null_masks[name] = missing + elif missing.any(): + values = list(values) + for index in np.flatnonzero(missing): + values[index] = None + return values + + result, peak = run_reads( + ((name, reader(name)) for name in names), storage.max_concurrency, storage.row_buffer_bytes + ) + storage._peak_row_buffer_bytes = peak + return result source_columns = _source_columns(table) with storage._owner.lock: storage._check_open() diff --git a/src/blosc2/ctable_storage.py b/src/blosc2/ctable_storage.py index 592a595db..f01bc4925 100644 --- a/src/blosc2/ctable_storage.py +++ b/src/blosc2/ctable_storage.py @@ -648,8 +648,12 @@ def index_anchor_path(self, col_name: str) -> str | None: def column_cbytes_for_info(column): - """Do not present shared HDF5 record storage as a per-field compressed size.""" - return None if isinstance(column, _RemoteHDF5Field) else getattr(column, "cbytes", None) + """Hide compressed sizes unavailable for remote fields.""" + return ( + None + if isinstance(column, _RemoteHDF5Field) or getattr(column, "_compressed_size_unavailable", False) + else getattr(column, "cbytes", None) + ) class _RemoteHDF5Field(blosc2.Operand): diff --git a/src/blosc2/remote_ctable.py b/src/blosc2/remote_ctable.py index e2909fdb6..1623e70b6 100644 --- a/src/blosc2/remote_ctable.py +++ b/src/blosc2/remote_ctable.py @@ -9,8 +9,9 @@ from __future__ import annotations import operator -import os # noqa: TC003 +import os +import blosc2 from blosc2.ctable import CTable from blosc2.ctable_storage import RemoteTableStorage from blosc2.remote_array import CACHE_POLICY_DEFAULT, RemoteMetadataMapping @@ -86,14 +87,60 @@ def __new__( cache_policy=CACHE_POLICY_DEFAULT, max_cache_bytes=CACHE_POLICY_DEFAULT, cache_dir=None, + shared_cache=False, hdf5_index=None, max_concurrency=8, metadata_buffer_bytes=8 << 20, row_buffer_bytes=64 << 20, + source_format=None, + parquet_options=None, + columns=None, + max_rows=None, + string_max_length=None, + null_storage=None, + auto_null_sentinels=True, + separate_nested_cols=True, + list_serializer="msgpack", + blosc2_batch_size=2048, + blosc2_items_per_block=None, + batch_size=2048, + cparams=None, + dparams=None, + validate=False, _filesystem=None, _filesystem_resolver=None, _batch_validator=None, ): + parquet = source_format == "parquet" or ( + isinstance(urlpath, (str, os.PathLike)) + and os.fspath(urlpath).split("?", 1)[0].lower().endswith(".parquet") + ) + if parquet and source_format not in (None, "parquet"): + raise ValueError("source_format conflicts with the .parquet suffix") + if not parquet and ( + source_format is not None + or any( + value is not None + for value in ( + parquet_options, + columns, + max_rows, + string_max_length, + null_storage, + blosc2_items_per_block, + cparams, + dparams, + ) + ) + or auto_null_sentinels is not True + or separate_nested_cols is not True + or list_serializer != "msgpack" + or blosc2_batch_size != 2048 + or batch_size != 2048 + or validate is not False + or shared_cache + ): + raise TypeError("Parquet conversion options require a Parquet source") if urlpath is None: raise TypeError("RemoteCTable requires a remote B2Z URL") settings = { @@ -107,20 +154,61 @@ def __new__( from blosc2.remote_store import RemoteStore - store = RemoteStore( - urlpath, - dataset=dataset, - path=path, - storage_options=storage_options, - cache_policy=cache_policy, - max_cache_bytes=max_cache_bytes, - cache_dir=cache_dir, - hdf5_index=hdf5_index, - _allow_array_root=True, - _filesystem=_filesystem, - _filesystem_resolver=_filesystem_resolver, - _batch_validator=_batch_validator, - ) + conversion = None + if parquet: + conversion = { + "parquet_options": parquet_options, + "columns": columns, + "max_rows": max_rows, + "string_max_length": string_max_length, + "null_storage": null_storage, + "auto_null_sentinels": auto_null_sentinels, + "separate_nested_cols": separate_nested_cols, + "list_serializer": list_serializer, + "blosc2_batch_size": blosc2_batch_size, + "blosc2_items_per_block": blosc2_items_per_block, + "batch_size": batch_size, + "cparams": cparams, + "dparams": dparams, + "validate": validate, + } + + if shared_cache: + if cache_dir is None or ( + cache_policy is not CACHE_POLICY_DEFAULT and cache_policy is not blosc2.CachePolicy.DISK + ): + raise ValueError("shared_cache=True requires a disk cache") + store = RemoteStore.with_sparse_cache( + urlpath, + cache_dir, + dataset=dataset, + path=path, + storage_options=storage_options, + max_cache_bytes=max_cache_bytes, + _filesystem=_filesystem, + _filesystem_resolver=_filesystem_resolver, + _batch_validator=_batch_validator, + _source_format="parquet" if parquet else None, + _parquet_conversion=conversion, + ) + else: + store = RemoteStore( + urlpath, + dataset=dataset, + path=path, + storage_options=storage_options, + cache_policy=cache_policy, + max_cache_bytes=max_cache_bytes, + cache_dir=cache_dir, + hdf5_index=hdf5_index, + _allow_array_root=True, + _filesystem=_filesystem, + _filesystem_resolver=_filesystem_resolver, + _batch_validator=_batch_validator, + _source_format="parquet" if parquet else None, + _parquet_conversion=conversion, + _allow_local_source=parquet, + ) try: _, full = store._resolve("") kind, diagnostic = store._owner.nodes[full] @@ -136,6 +224,19 @@ def __init__(self, *args, **kwargs): # Construction is completed by CTable._open_from_storage() in __new__. pass + @classmethod + def open_reference(cls, path, *, storage_options=None, parquet_options=None): + """Reopen a saved Parquet RemoteStore archive.""" + if parquet_options is not None: + raise TypeError("Parquet reader options are frozen in the saved reference") + from blosc2.remote_store import RemoteStore + + table = RemoteStore._open_artifact(path, storage_options=storage_options) + if not isinstance(table, cls): + table.close() + raise ValueError("Reference does not contain a remote CTable") + return table + @classmethod def with_sparse_cache( cls, @@ -157,6 +258,7 @@ def with_sparse_cache( _source_validator=None, _manifest_validator=None, _max_nodes=None, + source_format=None, ): """Attach a remote CTable to a sparse disk cache shared across processes. @@ -191,6 +293,7 @@ def with_sparse_cache( _source_validator=_source_validator, _manifest_validator=_manifest_validator, _max_nodes=_max_nodes, + _source_format=source_format, ) try: _, full = store._resolve("") @@ -206,7 +309,14 @@ def with_sparse_cache( @classmethod def _from_owner(cls, owner, full_path, **settings): settings = {name: _positive_integer(name, value) for name, value in settings.items()} - storage = RemoteTableStorage(owner, full_path, **settings) + if owner.format == "parquet": + from blosc2.remote_parquet import ParquetTableStorage + + storage = ParquetTableStorage( + owner, owner.parquet_schema, owner.parquet_physical, owner.parquet_length, **settings + ) + else: + storage = RemoteTableStorage(owner, full_path, **settings) try: return cls._open_from_storage(storage) except BaseException: @@ -295,10 +405,12 @@ def attrs(self): @property def source(self): storage = self._remote_storage() + from blosc2.remote_store import public_source_url + return { "kind": storage._owner.format, "version": 1, - "urlpath": storage._owner.urlpath, + "urlpath": public_source_url(storage._owner.urlpath), "dataset": storage._root_key, "assume_immutable": True, } diff --git a/src/blosc2/remote_parquet.py b/src/blosc2/remote_parquet.py new file mode 100644 index 000000000..235809dc6 --- /dev/null +++ b/src/blosc2/remote_parquet.py @@ -0,0 +1,900 @@ +"""Read-only, row-group-backed Parquet tables.""" + +from __future__ import annotations + +import hashlib +import math +import os +import re +import shutil +import uuid +import zipfile +from collections import OrderedDict +from contextlib import ExitStack, contextmanager +from dataclasses import asdict +from enum import Enum +from types import SimpleNamespace +from urllib.parse import urlsplit + +import numpy as np + +import blosc2 +from blosc2.ctable import CTable, NullPolicy, get_null_policy, null_policy +from blosc2.ctable_storage import RemoteTableStorage, _AllValidRows, split_field_path +from blosc2.schema_compiler import schema_from_dict, schema_to_dict + + +def _portable(value): + if isinstance(value, blosc2.CParams | blosc2.DParams | NullPolicy): + value = asdict(value) + if isinstance(value, Enum): + value = value.value + if isinstance(value, np.generic): + return _portable(value.item()) + if value is None or isinstance(value, bool | int | float | str | bytes): + return value + if isinstance(value, (list, tuple)): + return [_portable(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {key: _portable(item) for key, item in value.items()} + raise TypeError(f"Nonportable Parquet reader or conversion option: {type(value).__name__}") + + +def parquet_identity(conversion): + import msgpack + import pyarrow + + def stable(value): + if isinstance(value, float) and not math.isfinite(value): + return str(value) + if isinstance(value, dict): + return {key: stable(item) for key, item in value.items()} + if isinstance(value, list): + return [stable(item) for item in value] + return value + + defaults = { + "parquet_options": None, + "columns": None, + "max_rows": None, + "string_max_length": None, + "null_storage": None, + "auto_null_sentinels": True, + "separate_nested_cols": True, + "list_serializer": "msgpack", + "blosc2_batch_size": 2048, + "blosc2_items_per_block": None, + "batch_size": 2048, + "cparams": None, + "dparams": None, + "validate": False, + } + options = defaults | {key: value for key, value in conversion.items() if key != "_effective_null_policy"} + identity = { + "version": 1, + "blosc2": blosc2.__version__, + "pyarrow": pyarrow.__version__, + "conversion": stable(_portable(options)), + "null_policy": stable( + _portable(asdict(conversion.get("_effective_null_policy") or get_null_policy())) + ), + } + return hashlib.sha256(msgpack.packb(identity, use_bin_type=True)).hexdigest() + + +def validate_parquet_metadata(metadata): + if not isinstance(metadata, dict) or not isinstance(metadata.get("parquet"), dict): + raise ValueError("Missing Parquet discovery metadata") + retained = metadata["parquet"] + discovery = retained.get("discovery") + if not isinstance(discovery, dict) or discovery.get("version") != 1: + raise ValueError("Invalid Parquet discovery metadata") + for key in ("schema", "physical", "options"): + if not isinstance(discovery.get(key), dict): + raise ValueError(f"Invalid Parquet {key}") + if not isinstance(discovery.get("arrow_metadata"), bytes): + raise ValueError("Invalid Parquet footer metadata") + if not isinstance(discovery.get("row_ends"), list) or any( + isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in discovery["row_ends"] + ): + raise ValueError("Invalid Parquet row groups") + if not isinstance(discovery.get("length"), int) or discovery["length"] < 0: + raise ValueError("Invalid Parquet length") + if not isinstance(retained.get("reader_options"), dict): + raise ValueError("Invalid Parquet reader options") + if not isinstance(retained.get("conversion"), dict) or not isinstance( + retained.get("source_marker"), dict + ): + raise ValueError("Invalid Parquet conversion or source marker") + + +def _restore_compression_options(options): + for name, cls in (("cparams", blosc2.CParams), ("dparams", blosc2.DParams)): + if isinstance(options.get(name), dict): + options[name] = cls(**options[name]) + + +def _parquet_discovery(owner, schema, physical, length): + import pyarrow as pa + + sink = pa.BufferOutputStream() + owner.arrow_metadata.write_metadata_file(sink) + options = owner.parquet_options + return { + "version": 1, + "arrow_metadata": sink.getvalue().to_pybytes(), + "schema": schema_to_dict(schema), + "physical": physical, + "row_ends": owner.row_ends.tolist(), + "length": length, + "options": { + **_portable({key: value for key, value in options.items() if key != "null_policy"}), + "null_policy": asdict(options["null_policy"]), + }, + } + + +class _CountingHandle: + def __init__(self, handle, traffic): + self.handle = handle + self.traffic = traffic + self.ranges = () + + @contextmanager + def buffered_ranges(self, ranges): + previous, self.ranges = self.ranges, ranges + try: + yield + finally: + self.ranges = previous + + def read(self, size=-1): + start = self.handle.tell() + for offset, data in self.ranges: + if size >= 0 and offset <= start and start + size <= offset + len(data): + self.handle.seek(start + size) + return data[start - offset : start - offset + size] + data = self.handle.read(size) + self.traffic.charge(len(data)) + return data + + def readinto(self, buffer): + data = self.read(len(buffer)) + buffer[: len(data)] = data + return len(data) + + def __getattr__(self, name): + return getattr(self.handle, name) + + +def _source_marker(urlpath, storage_options, filesystem=None): + import fsspec + from fsspec.asyn import sync + + fs, path = ( + (filesystem, urlpath) + if filesystem is not None + else fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + ) + if urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session"): + try: + info = sync(fs.loop, _http_source_info, fs, path) + except (OSError, RuntimeError): + info = {} + if info.get("size") is None: + info = {**info, **fs.info(path)} + else: + info = fs.info(path) + return { + key: str(info[key]) + for key in ( + "size", + "etag", + "ETag", + "version_id", + "mtime", + "created", + "LastModified", + "Last-Modified", + "x-bz-file-id", + ) + if info.get(key) is not None + } + + +async def _http_source_info(fs, path): + kwargs = fs.kwargs.copy() + allow_redirects = kwargs.pop("allow_redirects", True) + headers = kwargs.pop("headers", {}).copy() + headers["Accept-Encoding"] = "identity" + session = await fs.set_session() + async with session.head( + fs.encode_url(path), headers=headers, allow_redirects=allow_redirects, **kwargs + ) as response: + fs._raise_not_found_for_status(response, path) + info = {} + for key in ("Content-Length", "ETag", "Last-Modified", "x-bz-file-id"): + if response.headers.get(key): + if key == "Content-Length" and response.headers.get("Content-Encoding", "identity") not in { + "", + "identity", + }: + continue + info["size" if key == "Content-Length" else key] = response.headers[key] + return info + + +def _open_source_handle(urlpath, storage_options, marker, traffic, filesystem=None): + import fsspec + + fs, path = ( + (filesystem, urlpath) + if filesystem is not None + else fsspec.core.url_to_fs(urlpath, **(storage_options or {})) + ) + kwargs = ( + {"size": int(marker["size"])} + if marker and urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session") + else {} + ) + if urlsplit(urlpath).scheme in {"http", "https"} and hasattr(fs, "set_session"): + kwargs["cache_type"] = (storage_options or {}).get("cache_type", "none") + return _CountingHandle(fs.open(path, "rb", **kwargs), traffic) + + +def _group_filename(number, physical): + token = hashlib.sha256(physical.encode()).hexdigest()[:16] + return f"{number}-{token}.b2d" + + +def valid_group_name(name): + return isinstance(name, str) and re.fullmatch(r"[0-9]+-[0-9a-f]{16}\.b2d", name) is not None + + +def _projected_field(name, physical, paths): + parts = split_field_path(name) + matches = set() + for path in paths: + segments = path.split(".") + for start in range(len(segments) - len(parts) + 1): + if tuple(segments[start : start + len(parts)]) != parts: + continue + prefix = ".".join(segments[: start + len(parts)]) + if not physical or prefix == physical or prefix.startswith(f"{physical}."): + matches.add(prefix) + return matches.pop() if len(matches) == 1 else physical + + +def discover_parquet(owner, conversion): # noqa: C901 + """Build the one CTable root and keep only the Arrow reader on the store owner.""" + import pyarrow as pa + import pyarrow.parquet as pq + + if owner.root: + raise ValueError("Parquet files support only the root table") + retained = owner.metadata.get("parquet") + if retained is not None: + try: + discovery = retained["discovery"] + schema = schema_from_dict(discovery["schema"]) + physical = discovery["physical"] + ends = np.asarray(discovery["row_ends"], dtype=np.int64) + metadata = pq.read_metadata(pa.BufferReader(discovery["arrow_metadata"])) + if ( + discovery["version"] != 1 + or set(physical) != {column.name for column in schema.columns} + or ends.ndim != 1 + or len(ends) > metadata.num_row_groups + or np.any(ends < 0) + or np.any(ends[1:] < ends[:-1]) + or not isinstance(discovery["length"], int) + or discovery["length"] < 0 + ): + raise ValueError("Invalid retained Parquet metadata") + options = dict(discovery["options"]) + options["null_policy"] = NullPolicy(**options["null_policy"]) + _restore_compression_options(options) + except (KeyError, TypeError, ValueError, OSError) as exc: + raise ValueError("Invalid retained Parquet metadata") from exc + owner.arrow_metadata = metadata + validate_parquet_groups(owner, metadata) + owner.row_ends = ends + owner.parquet_schema = schema + owner.parquet_physical = physical + owner.parquet_length = discovery["length"] + owner.parquet_options = options + owner.parquet_reader_options = retained.get("reader_options") or {} + owner.parquet_source_marker = retained.get("source_marker") + owner.nodes[""] = ("ctable", {"kind": "ctable"}) + return + + reader_options = conversion.get("parquet_options") or {} + if reader_options.get("memory_map"): + raise ValueError("memory_map is incompatible with a remote Parquet handle") + columns = conversion.get("columns") + if columns is not None and len(set(columns)) != len(columns): + raise ValueError("columns must be unique") + max_rows = conversion.get("max_rows") + if max_rows is not None and max_rows < 0: + raise ValueError("max_rows must be non-negative") + batch_size = conversion.get("batch_size", 2048) + CTable._validate_arrow_batch_size(batch_size) + policy = conversion.get("_effective_null_policy") or get_null_policy() + options = { + "null_policy": policy, + "string_max_length": conversion.get("string_max_length"), + "auto_null_sentinels": conversion.get("auto_null_sentinels", True), + "null_storage": conversion.get("null_storage"), + "separate_nested_cols": conversion.get("separate_nested_cols", True), + "list_serializer": conversion.get("list_serializer", "msgpack"), + "blosc2_batch_size": conversion.get("blosc2_batch_size", 2048), + "blosc2_items_per_block": conversion.get("blosc2_items_per_block"), + "batch_size": batch_size, + "cparams": conversion.get("cparams"), + "dparams": conversion.get("dparams"), + "validate": conversion.get("validate", False), + } + marker = ( + _source_marker(owner.urlpath, owner.storage_options, owner.filesystem) + if owner.parquet_persistent + else None + ) + if marker is not None and ("size" not in marker or len(marker) < 2): + raise ValueError("A persistent Parquet cache requires source size and a version marker") + handle = _open_source_handle( + owner.urlpath, owner.storage_options, marker, owner.traffic, owner.filesystem + ) + pf = None + try: + pf = pq.ParquetFile(handle, **reader_options) + fields = pf.schema_arrow + if columns is not None: + fields = pa.schema([fields.field(name) for name in columns]) + root_name = "root" + if "" in fields.names: + while root_name in fields.names: + root_name += "_1" + flatten_root = options["separate_nested_cols"] and CTable._detect_unnamed_root_list_struct( + pa, fields + ) + sample_needed = (options["null_storage"] or policy.resolve_null_storage()) != "mask" + sample = ( + next(pf.iter_batches(batch_size=1 if flatten_root else batch_size, columns=columns), None) + if sample_needed + else None + ) + if sample is None: + sample = pa.RecordBatch.from_arrays([pa.array([], type=f.type) for f in fields], schema=fields) + if not flatten_root and "" in fields.names: + sample = sample.rename_columns([root_name if name == "" else name for name in fields.names]) + meta = dict(sample.schema.metadata or {}) + meta[b"blosc2_empty_root_physical"] = root_name.encode() + sample = sample.replace_schema_metadata(meta) + options.update(root_name=root_name, flatten_root=flatten_root) + with null_policy(policy): + probe = CTable.from_arrow( + sample.schema, + [sample], + **{ + key: options[key] + for key in ( + "string_max_length", + "auto_null_sentinels", + "null_storage", + "separate_nested_cols", + "list_serializer", + "blosc2_batch_size", + "blosc2_items_per_block", + "cparams", + "dparams", + "validate", + ) + }, + ) + physical = { + name: ( + "" + if flatten_root or (split_field_path(name)[0] == root_name and "" in fields.names) + else name + if name in fields.names + else split_field_path(name)[0] + ) + for name in probe.col_names + } + paths = [pf.schema.column(i).path for i in range(len(pf.schema.names))] + physical = { + name: _projected_field(name, source, paths) + if flatten_root or (source and source != name) + else source + for name, source in physical.items() + } + if flatten_root: + lengths = [] + count = 0 + for group in range(pf.num_row_groups): + if max_rows is not None and count >= max_rows: + break + row_group = pf.metadata.row_group(group) + leaf = min( + (row_group.column(i) for i in range(row_group.num_columns)), + key=lambda column: column.total_compressed_size, + ) + arr = pf.read_row_group(group, columns=[leaf.path_in_schema]).column(0) + size = pa.compute.sum(pa.compute.list_value_length(arr)).as_py() or 0 + lengths.append(size) + count += size + ends = np.cumsum(lengths) + else: + ends = np.cumsum([pf.metadata.row_group(i).num_rows for i in range(pf.num_row_groups)]) + length = int(ends[-1]) if len(ends) else 0 + if max_rows is not None: + length = min(length, max_rows) + owner.parquet_handle = handle + owner.parquet_file = pf + owner.arrow_metadata = pf.metadata + validate_parquet_groups(owner, pf.metadata) + owner.parquet_schema = probe._schema + owner.parquet_physical = physical + owner.parquet_length = length + owner.parquet_options = options + owner.parquet_reader_options = reader_options + owner.parquet_source_marker = marker + owner.row_ends = ends + discovery = _parquet_discovery(owner, probe._schema, physical, length) + owner.metadata["parquet"] = { + "discovery": discovery, + "reader_options": _portable(reader_options), + "source_marker": marker, + "conversion": _portable(conversion), + } + owner.nodes[""] = ("ctable", {"kind": "ctable"}) + probe.close() + except BaseException: + if pf is not None: + pf.close() + handle.close() + raise + + +def validate_parquet_groups(owner, footer): + if owner.source_validator is None: + return + for number in range(footer.num_row_groups): + size = max(1, footer.row_group(number).total_byte_size) + geometry = SimpleNamespace(shape=(size,), dtype=np.dtype("u1"), chunks=(size,), blocks=(size,)) + owner.source_validator(geometry) + + +class ParquetCache: + """One cache leaf for converted Parquet row groups in a RemoteStore.""" + + def __init__(self, owner): + self.owner = owner + self._cache_key = "parquet" + self._cache_sizes = {} + self._cache_lru = OrderedDict() + self.cache = OrderedDict() + self.cache_bytes = 0 + self.cache_dir = ( + owner.disk.path / f"{owner.generation}.b2d" / "parquet-groups" + if owner.disk is not None + else getattr(owner, "parquet_artifact_dir", None) + ) + if self.cache_dir is not None: + self.cache_dir.mkdir(parents=True, exist_ok=True) + owner.cache_coordinator.register(self) + self._sync_evictions() + + _filename = staticmethod(_group_filename) + + def _sync_evictions(self): + if self.cache_dir is None: + return + entries = sorted(self.cache_dir.glob("*-*.b2d"), key=lambda path: path.stat().st_mtime_ns) + present = {path.name for path in entries} + for name in (key for key in tuple(self._cache_sizes) if isinstance(key, str)): + if name not in present: + self._cache_sizes.pop(name) + self._cache_lru.pop(name, None) + self.owner.cache_coordinator.forget(self, name) + for path in entries: + if path.name not in self._cache_sizes: + self._cache_sizes[path.name] = sum( + file.stat().st_size for file in path.rglob("*") if file.is_file() + ) + self._cache_lru[path.name] = None + if self.owner.is_mutable: + self.owner.cache_coordinator.touch(self, path.name) + + def _retained_cache_bytes(self): + return sum(self._cache_sizes.values()) + + def _trim_cache(self, target_bytes, *, max_chunks=None): + evicted = [] + while ( + self.cache_dir is not None + and self.owner.is_mutable + and self._retained_cache_bytes() > target_bytes + and self._cache_lru + and (max_chunks is None or len(evicted) < max_chunks) + ): + name = next(iter(self._cache_lru)) + shutil.rmtree(self.cache_dir / name) + self._cache_lru.pop(name) + self._cache_sizes.pop(name) + self.owner.cache_coordinator.forget(self, name) + evicted.append(name) + while ( + self._retained_cache_bytes() > target_bytes + and self.cache + and (max_chunks is None or len(evicted) < max_chunks) + ): + key, (table, size) = self.cache.popitem(last=False) + self.cache_bytes -= size + self._cache_sizes.pop(key, None) + self._cache_lru.pop(key, None) + self.owner.cache_coordinator.forget(self, key) + table.close() + evicted.append(key) + return tuple(evicted) + + def _ensure_reader(self): + owner = self.owner + if owner.parquet_file is None: + import pyarrow.parquet as pq + + if owner.artifact_path is not None and owner.parquet_source_marker is not None: + marker = _source_marker(owner.urlpath, owner.storage_options, owner.filesystem) + if marker != owner.parquet_source_marker: + raise RuntimeError("Parquet reference source has changed") + owner.parquet_handle = _open_source_handle( + owner.urlpath, + owner.storage_options, + owner.parquet_source_marker, + owner.traffic, + owner.filesystem, + ) + try: + owner.parquet_file = pq.ParquetFile( + owner.parquet_handle, + metadata=owner.arrow_metadata, + **owner.parquet_reader_options, + ) + except BaseException: + owner.parquet_handle.close() + owner.parquet_handle = None + raise + + def group_reads(self, number, physical): + """Fetch one missing physical field; caller owns decoding and cache changes.""" + owner = self.owner + path = None if self.cache_dir is None else self.cache_dir / _group_filename(number, physical) + if (number, physical) in self.cache or (path is not None and path.exists()): + return [] + if owner.cache_coordinator.cached_only: + return [] + self._ensure_reader() + handle = owner.parquet_handle.handle + if not hasattr(handle, "fs"): + return [] + group = owner.arrow_metadata.row_group(number) + ranges = [] + for index in range(group.num_columns): + column = group.column(index) + name = column.path_in_schema + if physical and name != physical and not name.startswith(physical + "."): + continue + offset = min( + value + for value in (column.dictionary_page_offset, column.data_page_offset) + if value is not None and value >= 0 + ) + size = column.total_compressed_size + if size < 0 or (handle.size is not None and offset + size > handle.size): + raise ValueError("Parquet column range exceeds source bounds") + ranges.append((offset, size)) + + def fetch(): + answers = [] + for offset, size in ranges: + if hasattr(handle, "async_fetch_range"): + # HTTPFile's transport checks range support without reading an ignored full response. + data = handle._fetch_range(offset, offset + size) + else: + data = handle.fs.cat_file(handle.path, start=offset, end=offset + size) + owner.traffic.charge(len(data)) + if len(data) != size: + raise OSError("Incomplete Parquet column range") + answers.append((offset, data)) + return answers + + return (yield fetch, (), sum(size for _, size in ranges)) if ranges else [] + + @contextmanager + def group(self, number, physical): + owner = self.owner + with owner.lock: + from blosc2.remote_store import CacheMiss + + key = (number, physical) + if key in self.cache: + self.cache.move_to_end(key) + owner.cache_coordinator.touch(self, key) + yield self.cache[key][0] + return + path = None if self.cache_dir is None else self.cache_dir / _group_filename(number, physical) + cache_file = path if path is not None and (owner.disk is not None or path.exists()) else None + if cache_file is not None and cache_file.exists(): + try: + table = CTable.open(cache_file) + start = 0 if number == 0 else int(owner.row_ends[number - 1]) + if len(table) == int(owner.row_ends[number]) - start: + cache_file.touch() + if owner.is_mutable: + owner.cache_coordinator.touch(self, cache_file.name) + try: + yield table + finally: + table.close() + return + table.close() + except (OSError, ValueError, RuntimeError, KeyError, TypeError, zipfile.BadZipFile): + pass + if owner.cache_coordinator.cached_only: + raise CacheMiss + self._ensure_reader() + arrow = owner.parquet_file.read_row_group(number, columns=[physical], use_threads=False) + options = owner.parquet_options + if physical == "" and not options["flatten_root"]: + arrow = arrow.rename_columns([options["root_name"]]) + meta = dict(arrow.schema.metadata or {}) + meta[b"blosc2_empty_root_physical"] = options["root_name"].encode() + arrow = arrow.replace_schema_metadata(meta) + with null_policy(options["null_policy"]): + table = CTable.from_arrow( + arrow.schema, + arrow.to_batches(max_chunksize=options["batch_size"]), + **{ + name: options[name] + for name in ( + "string_max_length", + "auto_null_sentinels", + "null_storage", + "separate_nested_cols", + "list_serializer", + "blosc2_batch_size", + "blosc2_items_per_block", + "cparams", + "dparams", + "validate", + ) + }, + capacity_hint=int(owner.row_ends[number]) + - (int(owner.row_ends[number - 1]) if number else 0), + ) + size = int(getattr(table, "cbytes", 0) or arrow.nbytes) + if cache_file is not None and owner.disk is not None: + temporary = cache_file.with_name(f".{uuid.uuid4().hex}.b2d") + try: + with table.copy(urlpath=temporary): + pass + if cache_file.exists(): + shutil.rmtree(cache_file) + os.replace(temporary, cache_file) + finally: + if temporary.exists(): + shutil.rmtree(temporary) + self._sync_evictions() + owner.cache_coordinator.touch(self, cache_file.name) + owner.cache_coordinator.enforce() + owner.save_manifest() + elif owner.cache_policy is blosc2.CachePolicy.MEMORY or ( + owner.disk is None and owner.cache_policy is not blosc2.CachePolicy.NONE + ): + pass + try: + yield table + finally: + if owner.cache_policy is blosc2.CachePolicy.MEMORY or ( + owner.disk is None and owner.cache_policy is not blosc2.CachePolicy.NONE + ): + self.cache[key] = table, size + self.cache_bytes += size + self._cache_sizes[key] = size + self._cache_lru[key] = None + owner.cache_coordinator.touch(self, key) + owner.cache_coordinator.enforce() + else: + table.close() + + def close(self): + for table, _ in self.cache.values(): + table.close() + self.cache.clear() + self._cache_sizes.clear() + self._cache_lru.clear() + + +class _ParquetColumn: + _compressed_size_unavailable = True + + def __init__(self, storage, name, mask=False): + self.storage = storage + self.name = name + self.mask = mask + self.shape = (storage.length,) + self.chunks = (min(max(storage.length, 1), 65536),) + self.blocks = self.chunks + self.dtype = np.dtype(bool) if mask else storage.schema.columns_by_name[name].dtype + self._dictionary = None + + def __len__(self): + return self.shape[0] + + @property + def nbytes(self): + return len(self) * self.dtype.itemsize if self.dtype is not None else 0 + + @property + def cbytes(self): + return 0 + + def flush(self): + pass + + @property + def dictionary(self): + with self.storage._owner.lock: + self.storage._check_open() + return self._dictionary_locked() + + def _dictionary_locked(self): + if self._dictionary is not None: + return self._dictionary + values = [] + seen = set() + for group in range(len(self.storage.row_ends)): + with self.storage._owner.group(group, self.storage.physical[self.name]) as table: + for value in table._cols[self.name].dictionary: + if value not in seen: + values.append(value) + seen.add(value) + self._dictionary = values + return values + + @property + def codes(self): + column = self + + class Codes: + def __getitem__(self, key): + mapping = {value: index for index, value in enumerate(column.dictionary)} + return np.asarray([mapping.get(value, -1) for value in column[key]], dtype=np.int32) + + return Codes() + + def __getitem__(self, key): + with self.storage._owner.lock: + return self._getitem_locked(key) + + def _getitem_locked(self, key): + from blosc2.ctable_remote_read import run_reads + + values, peak = run_reads(((self.name, self.reads(key)),), 1, self.storage.row_buffer_bytes) + self.storage._peak_row_buffer_bytes = peak + return values[self.name] + + def reads(self, key): + """Yield transport work while selection, decoding and caching stay serial.""" + self.storage._check_open() + if isinstance(key, tuple) and len(key) == 1: + key = key[0] + scalar = isinstance(key, (int, np.integer)) + positions = np.arange(*key.indices(self.shape[0])) if isinstance(key, slice) else np.asarray(key) + if scalar: + positions = np.asarray([int(key)]) + elif positions.dtype == np.dtype(bool): + if positions.ndim != 1 or len(positions) != self.shape[0]: + raise IndexError("boolean row mask has the wrong length") + positions = np.flatnonzero(positions) + positions = np.asarray(positions, dtype=np.int64).reshape(-1) + positions = np.where(positions < 0, positions + self.shape[0], positions) + if np.any((positions < 0) | (positions >= self.shape[0])): + raise IndexError("row index out of range") + values = [None] * len(positions) + groups = np.searchsorted(self.storage.row_ends, positions, side="right") + for group in np.unique(groups): + selected = np.flatnonzero(groups == group) + start = 0 if group == 0 else int(self.storage.row_ends[group - 1]) + local = positions[selected] - start + owner = self.storage._owner + physical = self.storage.physical[self.name] + ranges = yield from owner.parquet_cache.group_reads(int(group), physical) + # A retained group may be available without opening the source handle. + with ExitStack() as stack: + if ranges: + stack.enter_context(owner.parquet_handle.buffered_ranges(ranges)) + table = stack.enter_context(owner.group(int(group), physical)) + if self.mask: + mask = table._null_mask(self.name) + part = np.ones(len(local), dtype=bool) if mask is None else mask[local] + else: + part = table._cols[self.name][local] + del ranges + for target, value in zip(selected, part, strict=True): + values[int(target)] = value + result = ( + np.asarray(values, dtype=self.dtype) + if self.dtype is not None and self.dtype != np.dtype(object) + else np.asarray(values, dtype=object) + ) + return result[0] if scalar else result + + +class ParquetTableStorage(RemoteTableStorage): + def __init__( + self, + owner, + schema, + physical, + length, + *, + max_concurrency=8, + metadata_buffer_bytes=8 << 20, + row_buffer_bytes=64 << 20, + ): + self._owner = owner + self._generation = owner.generation + self._closed = False + self._root_key = "" + self.max_concurrency = max_concurrency + self.metadata_buffer_bytes = metadata_buffer_bytes + self.row_buffer_bytes = row_buffer_bytes + self.schema = schema + self.physical = physical + self.length = length + self.row_ends = owner.row_ends + owner.acquire() + + def check_kind(self): + self._check_open() + + def _metadata(self): + return {"kind": "ctable"} + + def load_user_attrs(self): + return {} + + def load_schema(self): + result = schema_to_dict(self.schema) + result["n_rows"] = self.length + return result + + def open_valid_rows(self): + return _AllValidRows(self.length, (65536,)) + + def open_columns(self, table, names, load): + for name in names: + load(name) + + def open_column(self, name): + return _ParquetColumn(self, name) + + open_list_column = open_column + + def open_dictionary_column(self, name, spec): + return self.open_column(name) + + def open_varlen_scalar_column(self, name, spec): + return self.open_column(name) + + def has_null_mask(self, name): + return bool(getattr(self.schema.columns_by_name[name].spec, "uses_mask", False)) + + def open_null_mask(self, name): + return _ParquetColumn(self, name, mask=True) + + def close(self): + if not self._closed: + self._closed = True + self._owner.release() diff --git a/src/blosc2/remote_store.py b/src/blosc2/remote_store.py index 19186e5b0..9c423dd81 100644 --- a/src/blosc2/remote_store.py +++ b/src/blosc2/remote_store.py @@ -31,6 +31,18 @@ RESERVED_NAMES = {"embed.b2e", "__vlmeta__"} +def public_source_url(url): + parsed = urlsplit(url) + if not parsed.scheme: + return url + clean = urlunsplit((parsed.scheme, parsed.netloc.rsplit("@", 1)[-1], parsed.path, parsed.query, "")) + try: + validate_persistable_url(clean) + except ValueError: + clean = urlunsplit((parsed.scheme, parsed.netloc.rsplit("@", 1)[-1], parsed.path, "", "")) + return clean + + class CacheMiss(Exception): """A cached-only table operation needs unavailable source data.""" @@ -83,7 +95,18 @@ def get_zip_offsets(zip_path: str) -> dict[str, dict[str, int]]: if not name or ":" in name: raise ValueError("Invalid RemoteStore archive path") if name != "embed.b2e": - RemoteDiscovery._validate(name) + if name.startswith("parquet-groups/"): + parts = name.split("/") + if ( + len(parts) < 3 + or any(part in {"", ".", ".."} for part in parts) + or any(char in name for char in "\\\0\n\r\t") + ): + raise ValueError("Invalid Parquet cache archive path") + else: + RemoteDiscovery._validate(name) + if (info.external_attr >> 16) & 0o170000 == 0o120000: + raise ValueError("RemoteStore archives cannot contain symbolic links") if info.filename in offsets or info.flag_bits & 1: raise ValueError("Invalid RemoteStore archive member") f.seek(info.header_offset) @@ -119,7 +142,7 @@ def _resolve_hdf5_options(hdf5_index, private_index, source_format): class RemoteDiscovery: """Shared metadata and source resources, independent of browser presentation.""" - def __init__( + def __init__( # noqa: C901 self, urlpath, storage_options=None, @@ -141,6 +164,7 @@ def __init__( _refresh_source=False, _b2z_blob=None, _local_source=False, + _parquet_conversion=None, ): self.urlpath, dataset, self.format = parse_container_url(urlpath, dataset) if _source_format is not None: @@ -185,6 +209,10 @@ def __init__( self.notice = None self.archive = None self.zstore = None + self.parquet_file = None + self.parquet_handle = None + self.parquet_cache = None + self.parquet_conversion = _parquet_conversion or {} self.hdf5_blob = None self.hdf5_source_cache_path = None if _hdf5_blob is not None: @@ -198,7 +226,21 @@ def __init__( self.disk = None self.generation = manifest["generation"] if manifest else uuid.uuid4().hex self.metadata = manifest["metadata"] if manifest else {} + if self.format == "parquet" and manifest: + from blosc2.ctable import NullPolicy + from blosc2.remote_parquet import _restore_compression_options + + self.parquet_conversion = dict(self.metadata["parquet"].get("conversion", {})) + self.parquet_conversion["_effective_null_policy"] = NullPolicy( + **self.metadata["parquet"]["discovery"]["options"]["null_policy"] + ) + _restore_compression_options(self.parquet_conversion) + elif self.format == "parquet": + from blosc2.ctable import get_null_policy + + self.parquet_conversion.setdefault("_effective_null_policy", get_null_policy()) self.persist_metadata = persist_metadata + self.parquet_persistent = persist_metadata self._external_filesystem = _filesystem self.source_validator = _source_validator self.batch_validator = _batch_validator @@ -233,6 +275,10 @@ def __init__( self._open_b2z() elif self.format == "hdf5": self._open_hdf5(_hdf5_index) + elif self.format == "parquet": + from blosc2.remote_parquet import discover_parquet + + discover_parquet(self, self.parquet_conversion) else: self._open_zarr() self._check_node_limit() @@ -243,7 +289,11 @@ def __init__( self.close() raise - def _restore_manifest(self, manifest): + def _restore_manifest(self, manifest): # noqa: C901 + if self.format == "parquet": + from blosc2.remote_parquet import validate_parquet_metadata + + validate_parquet_metadata(manifest["metadata"]) for path, entry in manifest["nodes"].items(): self._validate(path) if ( @@ -288,6 +338,10 @@ def _restore_manifest(self, manifest): self.nodes[path] = (kind, self.hdf5_index["datasets"][path]) elif self.format == "zarr" and self.zstore is None: self._open_zarr() + elif self.format == "parquet": + from blosc2.remote_parquet import discover_parquet + + discover_parquet(self, self.parquet_conversion) self._check_node_limit() def _validate_hdf5_index(self): @@ -319,6 +373,9 @@ def save_manifest(self): "metadata": self.metadata, "caches": sorted(self.caches), "batch_caches": sorted(self.batch_caches), + "parquet_caches": ( + sorted(self.parquet_cache._cache_sizes) if self.parquet_cache is not None else [] + ), "cache_policy": getattr(self, "cache_policy", blosc2.CachePolicy.DISK).value, "max_cache_bytes": getattr(self, "max_cache_bytes", None), "mutable": getattr(self, "mutable", False), @@ -869,6 +926,8 @@ def check_open(): def load_ctable_attrs(self, table_path): """Load one table's user attributes without opening its data arrays.""" + if self.format == "parquet": + return {} if table_path in self.attrs: return dict(self.attrs[table_path]) member = "/".join(part for part in (table_path, "_vlmeta.b2f") if part) @@ -963,7 +1022,31 @@ def release(self): if not self._users: self.close() - def restore_caches(self, manifest): + def restore_caches(self, manifest): # noqa: C901 + if self.format == "parquet": + from blosc2.remote_parquet import ParquetCache + + if self.parquet_cache is not None: + self.parquet_cache.close() + if self.artifact_path is not None and self.disk is None and manifest: + self._cleanup_dir = tempfile.TemporaryDirectory(prefix="parquet-artifact-") + target = Path(self._cleanup_dir.name) / "parquet-groups" + target.mkdir() + members = set(manifest.get("parquet_caches", [])) + if self.artifact_offsets is not None: + with zipfile.ZipFile(self.artifact_path) as archive: + for info in archive.infolist(): + parts = PurePosixPath(info.filename).parts + if len(parts) >= 3 and parts[0] == "parquet-groups" and parts[1] in members: + archive.extract(info, self._cleanup_dir.name) + else: + for name in members: + source = Path(self.artifact_path) / "parquet-groups" / name + if source.is_dir(): + shutil.copytree(source, target / name) + self.parquet_artifact_dir = target + self.parquet_cache = ParquetCache(self) + return if manifest: self.restoring = True for path in manifest["caches"]: @@ -1004,6 +1087,11 @@ def restore_caches(self, manifest): self.cache_coordinator.enforce() self.restoring = False + def group(self, number, physical): + if self.format != "parquet" or self.parquet_cache is None: + raise RuntimeError("Parquet cache is unavailable") + return self.parquet_cache.group(number, physical) + def get_cache(self, source, *, seed=None): key = next(path for path, value in self.sources.items() if value is source) coordinator_key = f"{self.cache_namespace}:{key}" if self.cache_namespace else key @@ -1104,6 +1192,7 @@ def prepare_refresh(self, kind): _source_cache_dir=self.source_cache_dir, _refresh_source=True, _local_source=self.local_source, + _parquet_conversion=self.parquet_conversion, ) try: if replacement.nodes[replacement.root][0] != kind: @@ -1116,6 +1205,8 @@ def prepare_refresh(self, kind): replacement.mutable = self.mutable replacement.restoring = True replacement.disk = self.disk + if replacement.format == "parquet": + replacement.restore_caches(None) replacement.hdf5_source_cache_path = self.hdf5_source_cache_path if self.hdf5_source_cache_path is not None and replacement.hdf5_blob is None: # A large replacement has no retained bytes, but must invalidate @@ -1125,6 +1216,8 @@ def prepare_refresh(self, kind): ).hexdigest() return replacement except BaseException: + if replacement.disk is self.disk: + replacement.disk = None replacement.close() raise @@ -1154,6 +1247,15 @@ def _close_resources(self): if self.zstore is not None: self.zstore.close() self.zstore = None + if getattr(self, "parquet_cache", None) is not None: + self.parquet_cache.close() + self.parquet_cache = None + if getattr(self, "parquet_file", None) is not None: + self.parquet_file.close() + self.parquet_file = None + if getattr(self, "parquet_handle", None) is not None: + self.parquet_handle.close() + self.parquet_handle = None for source in self.sources.values(): if isinstance(source, blosc2.HDF5NDSource): source.close() @@ -1195,7 +1297,7 @@ def _export_metadata(self): metadata["ranges"] = ranges return metadata - def save_selection( + def save_selection( # noqa: C901 self, full_path, destination: str | os.PathLike, @@ -1205,8 +1307,10 @@ def save_selection( overwrite: bool = False, ) -> str: """Export the current store or subtree to a portable .b2z reference archive.""" - if self.local_source: + if self.local_source and self.format != "parquet": raise ValueError("local-source caches cannot be exported as portable RemoteStore references") + if not self.local_source: + validate_persistable_url(self.urlpath) if not isinstance(include_cache, bool): raise TypeError("include_cache must be a boolean") if mutable is not None and not isinstance(mutable, bool): @@ -1214,6 +1318,17 @@ def save_selection( dest_abs, dest_dir = self._validate_save_destination(destination, overwrite) with self.lock: + if self.format == "parquet" and self.parquet_source_marker is None: + from blosc2.remote_parquet import _source_marker + + self.parquet_source_marker = _source_marker( + self.urlpath, self.storage_options, self.filesystem + ) + if "size" not in self.parquet_source_marker or len(self.parquet_source_marker) < 2: + raise ValueError( + "A portable Parquet reference requires source size and a version marker" + ) + self.metadata["parquet"]["source_marker"] = self.parquet_source_marker effective_mutable = self.mutable if mutable is None else mutable if include_cache: retained = self.cache_coordinator.cache_bytes @@ -1240,6 +1355,22 @@ def save_selection( exported_batches = self._export_batch_caches(full_path, staging_dir) if include_cache else [] + exported_parquet = [] + if self.format == "parquet" and include_cache: + for name, (table, _) in self.parquet_cache.cache.items(): + filename = self.parquet_cache._filename(*name) + destination = Path(staging_dir) / "parquet-groups" / filename + destination.parent.mkdir(parents=True, exist_ok=True) + with table.copy(urlpath=destination): + pass + exported_parquet.append(filename) + if self.parquet_cache.cache_dir is not None: + for path in self.parquet_cache.cache_dir.glob("*-*.b2d"): + if path.name not in exported_parquet and path.is_dir(): + destination = Path(staging_dir) / "parquet-groups" / path.name + shutil.copytree(path, destination) + exported_parquet.append(path.name) + linked_exports = self._export_linked_stores(full_path, staging_dir) if include_cache else {} exported_manifest = { @@ -1253,6 +1384,7 @@ def save_selection( "metadata": metadata, "caches": sorted(exported_caches), "batch_caches": sorted(exported_batches), + "parquet_caches": sorted(exported_parquet), "cache_policy": self.cache_policy.value, "max_cache_bytes": self.max_cache_bytes, "mutable": effective_mutable, @@ -1358,9 +1490,15 @@ def _collect_export_nodes(self, group_full, include_cache): "dataset": group_full, "kind": self.format, } + if self.local_source: + exported_source["local"] = True fingerprint = storage_options_fingerprint(getattr(self, "storage_options", None)) if fingerprint: exported_source["storage_options"] = fingerprint + if self.format == "parquet": + from blosc2.remote_parquet import parquet_identity + + exported_source["options"] = parquet_identity(self.parquet_conversion) if prefix: exported_nodes = { k: (v[0], v[1] if v[0] in {"ctable", "remote_store", "unsupported"} else None) @@ -1459,7 +1597,15 @@ class RemoteStore(RemoteObject): @classmethod def _try_open_artifact( - cls, urlpath, dataset, storage_options, cache_policy, max_cache_bytes, cache_dir, allow_array_root + cls, + urlpath, + dataset, + storage_options, + cache_policy, + max_cache_bytes, + cache_dir, + allow_array_root, + allow_table_root, ): if not os.path.exists(urlpath): return None @@ -1475,11 +1621,13 @@ def _try_open_artifact( "max_cache_bytes": None if max_cache_bytes is CACHE_POLICY_DEFAULT else max_cache_bytes, "cache_dir": cache_dir, "_allow_array_root": allow_array_root, + "_as_store": True, + "allow_table_root": allow_table_root, } return cls._open_artifact(urlpath, **kwargs) @staticmethod - def _cache_source(urlpath, dataset, source_format, storage_options): + def _cache_source(urlpath, dataset, source_format, storage_options, parquet_conversion=None): base_url, root, kind = parse_container_url(urlpath, dataset) local = not urlsplit(base_url).scheme or bool(os.path.splitdrive(base_url)[0]) if local: @@ -1494,6 +1642,10 @@ def _cache_source(urlpath, dataset, source_format, storage_options): fingerprint = storage_options_fingerprint(storage_options) if fingerprint: source["storage_options"] = fingerprint + if source["kind"] == "parquet": + from blosc2.remote_parquet import parquet_identity + + source["options"] = parquet_identity(parquet_conversion or {}) return source @staticmethod @@ -1512,7 +1664,7 @@ def _validate_nested_storage_options(value): if value is not None and not callable(value) and not isinstance(value, dict): raise TypeError("nested_storage_options must be a mapping or callable") - def __init__( + def __init__( # noqa: C901 self, urlpath, *, @@ -1539,6 +1691,7 @@ def __init__( nested_storage_options=None, _b2z_blob=None, _allow_local_source=False, + _parquet_conversion=None, ): dataset = blosc2.core.resolve_dataset_path(dataset, path) if not isinstance(urlpath, (str, os.PathLike)): @@ -1546,10 +1699,20 @@ def __init__( urlpath = os.fspath(urlpath) hdf5_index, _source_format = _resolve_hdf5_options(hdf5_index, _hdf5_index, _source_format) artifact = self._try_open_artifact( - urlpath, dataset, storage_options, cache_policy, max_cache_bytes, cache_dir, _allow_array_root + urlpath, + dataset, + storage_options, + cache_policy, + max_cache_bytes, + cache_dir, + _allow_array_root, + allow_table_root, ) if artifact is not None: - self._attach(artifact._owner, artifact._path) + try: + self._attach(artifact._owner, artifact._path) + finally: + artifact.close() return if dataset is not None and not isinstance(dataset, str): raise TypeError("dataset must be a string") @@ -1562,11 +1725,13 @@ def __init__( and (not urlsplit(base_url).scheme or os.path.splitdrive(base_url)[0]) and os.path.exists(base_url) ) - if not local_source: - validate_persistable_url(base_url) - else: + if local_source: base_url = os.path.abspath(base_url) urlpath = base_url + elif ( + _source_format != "parquet" and parse_container_url(base_url)[2] != "parquet" + ) or cache_policy is blosc2.CachePolicy.DISK: + validate_persistable_url(base_url) local_hdf5 = local_source and _source_format == "hdf5" disk = None source_cache_path = source_cache_marker = None @@ -1574,7 +1739,9 @@ def __init__( if cache_policy is blosc2.CachePolicy.DISK: from blosc2.remote_store_cache import StoreDiskCache - source = self._cache_source(urlpath, dataset, _source_format, storage_options) + source = self._cache_source( + urlpath, dataset, _source_format, storage_options, _parquet_conversion + ) disk = StoreDiskCache(cache_dir, source) try: manifest = disk.load() if disk is not None else manifest @@ -1612,6 +1779,7 @@ def __init__( _source_cache_dir=cache_dir if disk is not None else None, _b2z_blob=_b2z_blob, _local_source=local_source, + _parquet_conversion=_parquet_conversion, ) manifest = owner.restored_manifest owner.attach_hdf5_source_cache(source_cache_path, source_cache_marker) @@ -1813,6 +1981,7 @@ def with_sparse_cache( # noqa: C901 _traffic=None, _source_format=None, _hdf5_index=None, + _parquet_conversion=None, ): """Attach an immutable remote hierarchy to a cache shared across processes. @@ -1830,14 +1999,26 @@ def with_sparse_cache( # noqa: C901 dataset = blosc2.core.resolve_dataset_path(dataset, path) limit = normalize_cache_limit(blosc2.CachePolicy.DISK, max_cache_bytes) - base, root, kind = parse_container_url(urlpath, dataset) + base, root, kind = parse_container_url(os.fspath(urlpath), dataset) if _source_format is not None: kind = _source_format - validate_persistable_url(base) + local_parquet = kind == "parquet" and ( + not urlsplit(base).scheme or bool(os.path.splitdrive(base)[0]) + ) + if local_parquet: + base = os.path.abspath(base) + else: + validate_persistable_url(base) source = {"urlpath": base, "dataset": (root or "").strip("/"), "kind": kind} + if local_parquet: + source["local"] = True fingerprint = storage_options_fingerprint(storage_options) if fingerprint: source["storage_options"] = fingerprint + if kind == "parquet": + from blosc2.remote_parquet import parquet_identity + + source["options"] = parquet_identity(_parquet_conversion or {}) disk = SharedStoreCache(runtime_cache_path, source) with disk.guard(): current = disk.load() @@ -1869,6 +2050,8 @@ def with_sparse_cache( # noqa: C901 _traffic=_traffic, _source_format=kind, _hdf5_index=_hdf5_index, + _parquet_conversion=_parquet_conversion, + _local_source=local_parquet, ) owner.disk = disk owner.shared = True @@ -1876,6 +2059,20 @@ def with_sparse_cache( # noqa: C901 owner.max_cache_bytes = limit owner.cache_coordinator = CacheCoordinator(limit) try: + if seed_manifest is not None and kind == "parquet": + names = set(seed_manifest.get("parquet_caches", [])) + target = disk.path / f"{owner.generation}.b2d" + if os.path.isdir(carrier): + for name in names: + shutil.copytree( + Path(carrier) / "parquet-groups" / name, target / "parquet-groups" / name + ) + else: + with zipfile.ZipFile(carrier) as archive: + for info in archive.infolist(): + parts = PurePosixPath(info.filename).parts + if len(parts) >= 3 and parts[0] == "parquet-groups" and parts[1] in names: + archive.extract(info, target) owner.restore_caches(current) if seed_manifest is not None: for key in seed_manifest["caches"]: @@ -2017,6 +2214,23 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 for file in folder.glob("*.chunk") if file.stem.isdigit() ) + groups = [] + if manifest["source"]["kind"] == "parquet": + from blosc2.remote_parquet import valid_group_name + + folder = disk.path / f"{manifest['generation']}.b2d" / "parquet-groups" + for name in manifest.get("parquet_caches", []): + if not valid_group_name(name): + raise ValueError("Invalid cached Parquet group name") + path = folder / name + if path.is_dir() and not path.is_symlink(): + groups.append( + ( + name, + path, + sum(file.stat().st_size for file in path.rglob("*") if file.is_file()), + ) + ) nested = [] nested_root = disk.parent / "nested" if nested_root.is_dir() and not nested_root.is_symlink(): @@ -2041,6 +2255,7 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 total = ( sum(size for _, _, size in leaves) + sum(size for _, _, size in batches) + + sum(size for _, _, size in groups) + sum(size for _, _, size in nested) ) removed = [] @@ -2061,6 +2276,12 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 file.unlink() removed.append((key, int(file.stem))) total -= size + for name, path, size in groups: + if total <= target_bytes or len(removed) >= max_chunks: + break + shutil.rmtree(path) + removed.append(("parquet", name)) + total -= size for parent, source, size in nested: if total <= target_bytes or len(removed) >= max_chunks: break @@ -2072,6 +2293,11 @@ def trim_sparse_cache(runtime_cache_path, source, target_bytes, *, max_chunks=64 ) removed.extend((f"linked:{parent.name}:{key}", chunk) for key, chunk in evicted) total += remaining - size + if manifest["source"]["kind"] == "parquet" and any(key == "parquet" for key, _ in removed): + manifest["parquet_caches"] = [ + name for name in manifest.get("parquet_caches", []) if (folder / name).is_dir() + ] + disk.publish(manifest) return tuple(removed), total def read_cached(self, path, item=(), *, nchunk=None): @@ -2244,7 +2470,7 @@ def source(self): return { "kind": self._owner.format, "version": 1, - "urlpath": self._owner.urlpath, + "urlpath": public_source_url(self._owner.urlpath), "dataset": full, "assume_immutable": True, } @@ -2419,7 +2645,12 @@ def _load_artifact_manifest(cls, urlpath): source = manifest["source"] if not isinstance(source, dict) or not isinstance(source.get("urlpath"), str): raise ValueError("Invalid RemoteStore manifest source descriptor") - validate_persistable_url(source["urlpath"]) + if not ( + (source.get("kind") == "parquet" or os.path.isdir(urlpath)) + and source.get("local") + and os.path.isfile(source["urlpath"]) + ): + validate_persistable_url(source["urlpath"]) cls._validate_artifact_manifest(manifest) return manifest, artifact_offsets @@ -2433,12 +2664,27 @@ def _validate_artifact_manifest(manifest, _depth=0): # noqa: C901 source = manifest["source"] root = source.get("dataset", "") RemoteDiscovery._validate(root) - if source.get("kind") not in {"b2z", "hdf5", "zarr"}: + if source.get("kind") not in {"b2z", "hdf5", "zarr", "parquet"}: raise ValueError("Invalid RemoteStore source kind") for field in ("nodes", "attrs", "listed", "metadata"): if not isinstance(manifest.get(field), dict): raise ValueError(f"Invalid RemoteStore manifest {field}") nodes = manifest["nodes"] + if source.get("kind") == "parquet": + from blosc2.remote_parquet import validate_parquet_metadata + + required = {"urlpath", "dataset", "kind", "options"} + if ( + not required <= set(source) <= required | {"local", "storage_options"} + or not isinstance(source["options"], str) + or len(source["options"]) != 64 + or any(char not in "0123456789abcdef" for char in source["options"]) + or ("local" in source and source["local"] is not True) + ): + raise ValueError("Invalid Parquet source descriptor") + if root or set(nodes) != {""} or nodes[""][0] != "ctable": + raise ValueError("Parquet RemoteStore requires one root CTable") + validate_parquet_metadata(manifest["metadata"]) for path, entry in nodes.items(): RemoteDiscovery._validate(path) if ( @@ -2450,10 +2696,13 @@ def _validate_artifact_manifest(manifest, _depth=0): # noqa: C901 if entry[0] == "ctable": metadata = entry[1] if ( - source.get("kind") not in {"b2z", "hdf5"} + source.get("kind") not in {"b2z", "hdf5", "parquet"} or not isinstance(metadata, dict) or metadata.get("kind") not in {"ctable", b"ctable"} - or not isinstance(metadata.get("schema"), (str, bytes)) + or ( + source.get("kind") != "parquet" + and not isinstance(metadata.get("schema"), (str, bytes)) + ) or ( source.get("kind") == "hdf5" and ( @@ -2506,6 +2755,15 @@ def _validate_artifact_manifest(manifest, _depth=0): # noqa: C901 ) ): raise ValueError("Invalid cached RemoteStore batch") + parquet_caches = manifest.get("parquet_caches", []) + if not isinstance(parquet_caches, list) or (parquet_caches and source.get("kind") != "parquet"): + raise ValueError("Invalid cached Parquet groups") + from blosc2.remote_parquet import valid_group_name + + if any(not valid_group_name(name) for name in parquet_caches) or len(set(parquet_caches)) != len( + parquet_caches + ): + raise ValueError("Invalid cached Parquet group name") linked = manifest.get("linked", {}) if not isinstance(linked, dict): raise ValueError("Invalid nested RemoteStore artifacts") @@ -2548,6 +2806,7 @@ def _open_mutable_artifact(cls, urlpath, manifest, storage_options, cache_policy dataset=source_desc.get("dataset"), manifest=manifest, persist_metadata=True, + _local_source=source_desc.get("local", False), ) except BaseException: if cleanup_dir is not None: @@ -2609,6 +2868,7 @@ def _open_immutable_artifact( dataset=source_desc.get("dataset"), manifest=manifest, persist_metadata=False, + _local_source=source_desc.get("local", False), _traffic=_traffic, _filesystem=( _filesystem_resolver(source_desc["urlpath"]) if _filesystem_resolver is not None else None @@ -2665,7 +2925,7 @@ def _open_artifact(cls, urlpath, mode="r", **kwargs): if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") if cache_policy is blosc2.CachePolicy.NONE and ( - manifest.get("caches") or manifest.get("batch_caches") + manifest.get("caches") or manifest.get("batch_caches") or manifest.get("parquet_caches") ): raise ValueError( "Cannot reopen a warm RemoteStore artifact with CachePolicy.NONE; " @@ -2702,6 +2962,13 @@ def _open_artifact(cls, urlpath, mode="r", **kwargs): urlpath, manifest, artifact_offsets, storage_options, cache_policy, limit, cache_dir ) + if kwargs.get("_as_store"): + if owner.nodes[owner.root][0] == "ctable" and not kwargs.get("allow_table_root"): + owner.close() + raise ValueError("RemoteStore requires a group; use RemoteCTable for a table") + result = object.__new__(cls) + result._attach(owner, "") + return result return cls._select_artifact(owner, kwargs.get("dataset"), kwargs.get("max_concurrency")) @classmethod diff --git a/src/blosc2/remote_store_cache.py b/src/blosc2/remote_store_cache.py index 9efebca38..24e27379a 100644 --- a/src/blosc2/remote_store_cache.py +++ b/src/blosc2/remote_store_cache.py @@ -255,6 +255,15 @@ def __enter__(self): if owner.zstore is not None: owner.zstore.close() owner.zstore = None + if owner.parquet_cache is not None: + owner.parquet_cache.close() + owner.parquet_cache = None + if owner.parquet_file is not None: + owner.parquet_file.close() + owner.parquet_file = None + if owner.parquet_handle is not None: + owner.parquet_handle.close() + owner.parquet_handle = None owner.sources.clear() owner.source_descriptors.clear() owner.batch_caches.clear() diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 27a087368..234cb5d05 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2687,7 +2687,7 @@ def _validate_shared_cache_request(urlpath, shared_cache, kwargs): kwargs["lazy"] = True -def open( +def open( # noqa: C901 urlpath: str | pathlib.Path | blosc2.URLPath, mode: str = "r", offset: int = 0, @@ -2857,12 +2857,15 @@ def open( Pre-computed native HDF5 index, or a local path or remote fsspec URL to its JSON encoding. It must match the source HDF5 URL and dataset scope. Arrays, PyTables tables, and hierarchy stores are supported. - source_format: {None, "blosc2", "zarr", "hdf5", "b2z"}, optional + source_format: {None, "blosc2", "zarr", "hdf5", "b2z", "parquet"}, optional Format of a lazy remote source. A ``.zarr`` URL path component selects Zarr automatically; a ``.h5`` or ``.hdf5`` path selects HDF5 automatically; a ``.b2z`` path selects B2Z automatically. An explicit value supports - suffix-free paths. Zarr, HDF5 and B2Z sources automatically enable + suffix-free paths. Zarr, HDF5, B2Z and Parquet sources automatically enable ``lazy=True``. + parquet_options: dict, optional + PyArrow ``ParquetFile`` reader options for a Parquet source. Conversion + options such as ``columns`` and ``max_rows`` are passed separately. assume_immutable: bool, optional With ``lazy=True``, skip remote identity checks before reads. Defaults to ``True``. Local disk caches always assume immutable sources; set to a new path or @@ -2964,6 +2967,41 @@ def open( True """ dataset = blosc2.core.resolve_dataset_path(dataset, path) + if kwargs.get("source_format") == "parquet" or ( + isinstance(urlpath, (str, pathlib.Path)) + and str(urlpath).split("?", 1)[0].lower().endswith(".parquet") + ): + if kwargs.get("source_format") not in (None, "parquet"): + raise ValueError("source_format conflicts with the .parquet suffix") + if mode != "r" or offset or (dataset is not None and dataset.strip("/")): + raise ValueError("Parquet sources require mode='r', offset=0, and the root table") + + kwargs.pop("source_format", None) + lazy = kwargs.pop("lazy", True) + if shared_cache and lazy is False: + raise TypeError("shared_cache requires lazy Parquet access") + if shared_cache: + if kwargs.get("cache_dir") is None: + raise ValueError("shared_cache=True requires cache_dir") + kwargs["shared_cache"] = True + if lazy is False: + from blosc2.core import fsspec_open + + cache_keys = kwargs.keys() & { + "cache_dir", + "cache_policy", + "max_cache_bytes", + "max_concurrency", + "metadata_buffer_bytes", + "row_buffer_bytes", + } + if cache_keys: + raise TypeError(f"{', '.join(sorted(cache_keys))} require lazy Parquet access") + options = kwargs.pop("storage_options", None) + reader = kwargs.pop("parquet_options", None) or {} + with fsspec_open(str(urlpath), "rb", options) as handle: + return blosc2.CTable.from_parquet(handle, **kwargs, **reader) + return blosc2.RemoteCTable(str(urlpath), dataset=dataset, source_format="parquet", **kwargs) _reject_table_buffer_options(kwargs) _validate_shared_cache_request(urlpath, shared_cache, kwargs) if isinstance(urlpath, blosc2.URLPath): diff --git a/tests/ctable/test_remote_ctable.py b/tests/ctable/test_remote_ctable.py index 1d6218067..e6cd3d173 100644 --- a/tests/ctable/test_remote_ctable.py +++ b/tests/ctable/test_remote_ctable.py @@ -689,6 +689,8 @@ def test_remote_example_batch_columns(tmp_path, capsys): assert local["message"][47] is None assert local["tags"][53] is None assert local["region"][59] is None + for name, period in (("temperature", 17), ("humidity", 29), ("status", 41), ("note", 43)): + np.testing.assert_array_equal(np.flatnonzero(local[name].is_null()), np.arange(0, 100, period)) url = f"memory://{tmp_path.name}-example-batches.b2z" fsspec.filesystem("memory").pipe(url, path.read_bytes()) @@ -700,6 +702,112 @@ def test_remote_example_batch_columns(tmp_path, capsys): assert "Dictionary costs (codes first, then full vocabulary on first decode)" in output +@pytest.mark.parametrize(("format_flag", "suffix"), [("pytables", ".h5"), ("parquet", ".parquet")]) +@pytest.mark.parametrize("explicit", [False, True]) +def test_remote_example_external_formats(tmp_path, capsys, monkeypatch, format_flag, suffix, explicit): + import runpy + import sys + from pathlib import Path + + pytest.importorskip("tables" if format_flag == "pytables" else "pyarrow") + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + path = tmp_path / f"readings{suffix}" + args = [str(script), "--write", str(path), "--rows", "120", "--batch-size", "37"] + if explicit: + args.insert(1, f"--{format_flag}") + if format_flag == "pytables": + args += ["--full", "station_id"] + monkeypatch.setattr(sys, "argv", args) + assert example["main"]() == 0 + + if format_flag == "parquet": + import pyarrow as pa + import pyarrow.parquet as pq + + source = pq.read_table(path) + assert source.column_names == [field.name for field in dataclasses.fields(example["Reading"])] + assert pa.types.is_large_string(source.schema.field("note").type) + assert pa.types.is_large_string(source.schema.field("message").type) + assert source.schema.field("tags").type == pa.list_(pa.int16()) + assert pa.types.is_dictionary(source.schema.field("region").type) + assert source["temperature"].null_count == 8 + assert source["humidity"].null_count == 5 + assert source["status"].null_count == 3 + assert source["note"].null_count == 3 + assert source["message"].null_count == 3 + assert source["tags"].null_count == 3 + assert source["region"].null_count == 3 + row_group = pq.read_metadata(path).row_group(0) + for name in ("id", "temperature", "note", "message"): + column = next( + row_group.column(i) + for i in range(row_group.num_columns) + if row_group.column(i).path_in_schema == name + ) + assert "RLE_DICTIONARY" not in column.encodings + + url = f"memory://{tmp_path.name}-readings{suffix}" + fsspec.filesystem("memory").pipe(url, path.read_bytes()) + if format_flag == "parquet": + with blosc2.open(url, source_format="parquet") as remote: + assert remote.to_arrow().to_pylist() == source.to_pylist() + monkeypatch.setattr(sys, "argv", [str(script), *([f"--{format_flag}"] if explicit else []), url]) + assert example["main"]() == 0 + output = capsys.readouterr().out + assert "first ids: [42]" in output + assert f"[Format: {'PyTables/HDF5' if format_flag == 'pytables' else 'Parquet'}]" in output + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_remote_example_blosc2_flag_and_inference(tmp_path, capsys, monkeypatch, explicit): + import runpy + import sys + from pathlib import Path + + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + path = tmp_path / "readings.b2z" + flag = ["--blosc2"] if explicit else [] + monkeypatch.setattr(sys, "argv", [str(script), *flag, "--write", str(path), "--rows", "120"]) + assert example["main"]() == 0 + url = f"memory://{tmp_path.name}-readings.b2z" + fsspec.filesystem("memory").pipe(url, path.read_bytes()) + monkeypatch.setattr(sys, "argv", [str(script), *flag, url]) + assert example["main"]() == 0 + assert "[Format: Blosc2 B2Z" in capsys.readouterr().out + + +def test_remote_example_unknown_extension(tmp_path, monkeypatch, capsys): + import runpy + import sys + from pathlib import Path + + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + for args in (["--write", str(tmp_path / "readings.csv")], ["memory://readings.csv"]): + monkeypatch.setattr(sys, "argv", [str(script), *args]) + with pytest.raises(SystemExit, match="2"): + example["main"]() + assert "cannot infer format" in capsys.readouterr().err + + +def test_remote_example_url_extension_ignores_query_string(monkeypatch): + import runpy + import sys + from pathlib import Path + + script = Path(__file__).resolve().parents[2] / "examples/ctable/remote_handling.py" + example = runpy.run_path(str(script)) + selected = [] + monkeypatch.setitem( + example["main"].__globals__, "access_external_table", lambda args: selected.append(args.parquet) + ) + monkeypatch.setattr(sys, "argv", [str(script), "https://example.com/readings.parquet?version=1"]) + assert example["main"]() == 0 + assert selected == [True] + + def test_disk_cache_metadata_key_order(tmp_path, monkeypatch): local = blosc2.CTable(dataclasses.make_dataclass("Sample", [("x", int)]), [(i,) for i in range(20)]) url = remote_table_url(tmp_path, local) diff --git a/tests/ctable/test_remote_parquet.py b/tests/ctable/test_remote_parquet.py new file mode 100644 index 000000000..ea9544f87 --- /dev/null +++ b/tests/ctable/test_remote_parquet.py @@ -0,0 +1,1097 @@ +"""Parity and row-group isolation for the Parquet remote-table prototype.""" + +import http.server +import os +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from decimal import Decimal +from email.utils import formatdate + +import numpy as np +import pytest + +import blosc2 +from blosc2 import remote_parquet +from blosc2.core import cache_directory_name +from blosc2.ctable import CTable +from blosc2.schema_compiler import schema_to_dict + +fsspec = pytest.importorskip("fsspec") +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + + +def test_windows_cache_directory_uses_source_basename(): + assert cache_directory_name(r"C:\data\source.parquet", b"cache").startswith("source.parquet--") + + +def test_source_url_preserves_non_sensitive_query(): + from blosc2.remote_store import public_source_url + + assert ( + public_source_url("https://example.com/source.parquet?version=2#section") + == "https://example.com/source.parquet?version=2" + ) + assert public_source_url("https://example.com/source.parquet?token=secret") == ( + "https://example.com/source.parquet" + ) + + +@pytest.mark.parametrize( + "source", + [ + pa.table({"id": [1, 2, None, 4], "text": ["a", None, "é", ""]}), + pa.table( + { + "unsigned": pa.array([0, 1, None, 255], type=pa.uint8()), + "float": pa.array([1.0, float("nan"), None, 4.5], type=pa.float32()), + "flag": pa.array([True, None, False, True], type=pa.bool_()), + } + ), + pa.table( + { + "large_text": pa.array(["é", "", None, "xyz"], type=pa.large_string()), + "binary": pa.array([b"\x00x", b"", None, b"hi"], type=pa.large_binary()), + } + ), + pa.table({"time": pa.array([0, None, 1234, 5678], type=pa.timestamp("ms", tz="UTC"))}), + pa.table({"items": pa.array([[1, None], [], None, [4]], type=pa.list_(pa.int64()))}), + pa.table( + { + "record": pa.array( + [{"id": 1}, None, {"id": 3}, {"id": 4}], + type=pa.struct([("id", pa.int32())]), + ) + } + ), + pa.table({"category": pa.array(["a", "b", "a", "c"]).dictionary_encode()}), + pa.table({"cell": pa.array([[1, 2], [2, 3], [3, 4], [5, 6]], type=pa.list_(pa.int32(), 2))}), + pa.table( + { + "items": pa.array( + [[{"id": 1, "name": "a"}], None, [], [{"id": None, "name": "b"}]], + type=pa.list_(pa.struct([("id", pa.int32()), ("name", pa.string())])), + ) + } + ), + pa.table( + { + "trip": pa.array( + [ + {"begin": {"lon": 1.5, "when": 1000}}, + None, + {"begin": {"lon": None, "when": None}}, + {"begin": {"lon": 4.5, "when": 4000}}, + ], + type=pa.struct( + [("begin", pa.struct([("lon", pa.float64()), ("when", pa.timestamp("ms"))]))] + ), + ) + } + ), + pa.table({"id": pa.array([], type=pa.int32()), "text": pa.array([], type=pa.string())}), + pa.Table.from_arrays( + [pa.array([1, 2, 3, 4]), pa.array([5, 6, 7, 8]), pa.array([9, 10, 11, 12])], + names=["", "root", "foo.bar"], + ), + pa.Table.from_arrays( + [pa.array([[{"id": 1}], [], None, [{"id": 4}]], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ), + ], + ids=[ + "scalar", + "numeric-bool", + "large-string-binary", + "timestamp", + "list", + "struct", + "dictionary", + "fixed-cell", + "list-struct", + "nested-struct-timestamp", + "empty", + "escaped-root", + "flattened-root", + ], +) +def test_remote_parquet_matches_importer(source, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table(source, path, row_group_size=2) + fs = fsspec.filesystem("memory") + fs.pipe("/remote-parity.parquet", path.read_bytes()) + with blosc2.CTable.from_parquet(path) as eager, blosc2.open("memory:///remote-parity.parquet") as remote: + assert remote.col_names == eager.col_names + assert len(remote) == len(eager) + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + if "float" in source.column_names: + np.testing.assert_equal(remote.to_arrow().to_pylist(), eager.to_arrow().to_pylist()) + else: + assert remote.to_arrow().equals(eager.to_arrow()) + if "cell" not in source.column_names: + assert list(remote) == list(eager) + + +def test_narrow_read_uses_only_one_later_group(tmp_path): + path = tmp_path / "wide.parquet" + pq.write_table(pa.table({"one": list(range(12)), "two": list(range(12))}), path, row_group_size=3) + fs = fsspec.filesystem("memory") + fs.pipe("/remote-wide.parquet", path.read_bytes()) + with blosc2.open("memory:///remote-wide.parquet") as remote: + before = remote.traffic.requests + assert remote["one"][10] == 10 + assert remote.traffic.requests == before + 1 + assert remote["one"][10] == 10 + assert remote.traffic.requests == before + 1 + + +def test_info_without_loading_parquet_groups(tmp_path): + path = tmp_path / "info.parquet" + pq.write_table(pa.table({"id": [1, 2, 3], "text": ["a", "bb", "ccc"]}), path) + with blosc2.open(path, cache_dir=tmp_path / "cache") as remote: + before = remote.traffic.requests + info = dict(remote.info_items) + assert info["nbytes"] == "27 (27 B)" # Three int64 values plus the virtual validity mask. + assert info["cbytes"] == info["cratio"] == "n/a" + assert "id" in info["columns"] + assert remote.traffic.requests == before + + +def test_unnamed_root_without_flattening(tmp_path): + path = tmp_path / "root.parquet" + source = pa.Table.from_arrays( + [ + pa.array( + [[{"root": 1}], [], None, [{"root": 4}]], + type=pa.list_(pa.struct([("root", pa.int32())])), + ) + ], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.CTable.from_parquet(path, separate_nested_cols=False) as eager: + with blosc2.open(path, separate_nested_cols=False) as remote: + assert len(remote) == len(eager) == 4 + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_empty_flattened_root(tmp_path): + path = tmp_path / "empty-root.parquet" + source = pa.Table.from_arrays( + [pa.array([[], None, []], type=pa.list_(pa.struct([("id", pa.int32())])))], names=[""] + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.CTable.from_parquet(path) as eager, blosc2.open(path) as remote: + assert len(remote) == len(eager) == 0 + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_flattened_root_counts_from_smallest_leaf(tmp_path): + path = tmp_path / "wide-root.parquet" + source = pa.Table.from_arrays( + [ + pa.array( + [ + [{"id": 1, "blob": os.urandom(128_000)}], + [], + [{"id": 2, "blob": os.urandom(128_000)}], + [{"id": 3, "blob": os.urandom(128_000)}], + ] + ) + ], + names=[""], + ) + pq.write_table(source, path, row_group_size=2, compression="NONE") + with blosc2.open(path) as remote: + assert len(remote) == 3 + assert remote.traffic.requests == 3 # Footer and one leaf per group. + assert remote.traffic.nbytes < path.stat().st_size // 2 + before = remote.traffic.nbytes + assert remote["id"][-1] == 3 + assert remote.traffic.nbytes - before < path.stat().st_size // 2 + assert remote["blob"][-1] == source.column(0)[-1].as_py()[-1]["blob"] + + +@pytest.mark.parametrize("limit", [0, 2, 5]) +def test_flattened_root_max_rows(tmp_path, limit): + path = tmp_path / "root.parquet" + source = pa.Table.from_arrays( + [ + pa.array( + [[{"id": 1}, {"id": 2}], [], None, [{"id": 4}], [{"id": 5}, {"id": 6}]], + type=pa.list_(pa.struct([("id", pa.int32())])), + ) + ], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.CTable.from_parquet(path, max_rows=limit) as eager: + with blosc2.open(path, max_rows=limit) as remote: + assert len(remote) == len(eager) + assert remote.to_arrow().equals(eager.to_arrow()) + + +@pytest.mark.parametrize("serializer", ["msgpack", "arrow"]) +def test_nested_list_serializers(tmp_path, serializer): + path = tmp_path / "nested.parquet" + source = pa.table( + {"items": pa.array([[[1, None], []], None, [], [[4]]], type=pa.list_(pa.list_(pa.int32())))} + ) + pq.write_table(source, path, row_group_size=2) + options = {"list_serializer": serializer, "blosc2_batch_size": 2, "blosc2_items_per_block": 1} + with blosc2.CTable.from_parquet(path, **options) as eager, blosc2.open(path, **options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_null_policy_frozen_at_open(tmp_path): + path = tmp_path / "nullable.parquet" + pq.write_table(pa.table({"id": pa.array([1, None, 3], type=pa.int8())}), path, row_group_size=1) + with blosc2.null_policy(blosc2.NullPolicy(signed_int_strategy="max")): + eager = blosc2.CTable.from_parquet(path) + remote = blosc2.open(path) + try: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + finally: + eager.close() + remote.close() + + +def test_selection_and_materialization(tmp_path): + path = tmp_path / "selected.parquet" + target = tmp_path / "selected.b2z" + pq.write_table(pa.table({"a": [0, 1, 2, 3], "b": [10, 11, 12, 13]}), path, row_group_size=2) + with blosc2.open(path, columns=["b", "a"], max_rows=3) as remote: + assert remote.col_names == ["b", "a"] + assert [row.a for row in remote] == [0, 1, 2] + assert remote.where("a > 0").to_arrow().column("a").to_pylist() == [1, 2] + with remote.copy(urlpath=target) as materialized: + assert materialized.to_arrow().equals(remote.to_arrow()) + with blosc2.open(target) as reopened: + assert reopened.to_arrow().column("b").to_pylist() == [10, 11, 12] + + +def test_direct_constructor_and_settings(tmp_path): + path = tmp_path / "direct.parquet" + pq.write_table(pa.table({"a": [1, 2]}), path) + with blosc2.RemoteCTable(path, max_concurrency=2, row_buffer_bytes=1024) as remote: + assert remote["a"][:].tolist() == [1, 2] + assert remote.max_concurrency == 2 + assert remote.row_buffer_bytes == 1024 + + +def test_explicit_extensionless_source_and_projection(tmp_path): + path = tmp_path / "source" + pq.write_table(pa.table({"a": [1, 2], "b": [3, 4]}), path) + with blosc2.open(path, source_format="parquet", columns=["b"]) as remote: + assert remote.col_names == ["b"] + assert remote.to_arrow().column("b").to_pylist() == [3, 4] + with pytest.raises(KeyError): + blosc2.open(path, source_format="parquet", columns=["missing"]) + + +def test_unsupported_type_rejected_at_open(tmp_path): + path = tmp_path / "decimal.parquet" + pq.write_table(pa.table({"amount": pa.array([Decimal("1.00")], type=pa.decimal128(6, 2))}), path) + with pytest.raises(TypeError, match="No blosc2 spec"): + blosc2.open(path) + + +def test_invalid_source_options_fail_clearly(tmp_path): + path = tmp_path / "data.parquet" + pq.write_table(pa.table({"a": [1]}), path) + with pytest.raises(ValueError, match="mode='r'"): + blosc2.open(path, mode="a") + with pytest.raises(TypeError, match="require lazy"): + blosc2.open(path, lazy=False, cache_dir=tmp_path / "cache") + with pytest.raises(ValueError, match="requires cache_dir"): + blosc2.open(path, shared_cache=True) + with pytest.raises(TypeError, match="shared_cache requires lazy"): + blosc2.open(path, lazy=False, shared_cache=True, cache_dir=tmp_path / "cache") + with pytest.raises(ValueError, match="memory_map"): + blosc2.open(path, parquet_options={"memory_map": True}) + with pytest.raises(ValueError, match="conflicts"): + blosc2.open(path, source_format="blosc2") + + +def test_portable_reference_reopens_and_detects_source_change(tmp_path): + path = tmp_path / "source.parquet" + carrier = tmp_path / "source.b2z" + pq.write_table(pa.table({"id": pa.array([1, None, 3], type=pa.int8())}), path) + with blosc2.null_policy(blosc2.NullPolicy(signed_int_strategy="max")): + with blosc2.open(path) as remote: + remote.save(carrier, include_cache=False) + with blosc2.open(carrier) as reopened: + assert reopened.to_arrow().column("id").to_pylist() == [1, None, 3] + with blosc2.RemoteCTable.open_reference(carrier) as reopened: + assert reopened.to_arrow().column("id").to_pylist() == [1, None, 3] + output = subprocess.check_output( + [ + sys.executable, + "-c", + "import blosc2,sys; print(blosc2.open(sys.argv[1]).to_arrow().column('id').to_pylist())", + str(carrier), + ], + text=True, + ) + assert output.strip() == "[1, None, 3]" + pq.write_table(pa.table({"id": pa.array([4, 5], type=pa.int8())}), path) + with blosc2.open(carrier) as reopened: + with pytest.raises((RuntimeError, OSError), match="changed"): + reopened["id"][0] + + +def test_reference_retains_cached_group_and_runtime_options(tmp_path): + path = tmp_path / "source.parquet" + carrier = tmp_path / "source.b2z" + pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) + with blosc2.open(path, storage_options={"auto_mkdir": True}) as remote: + assert remote["id"][5] == 5 + remote.save(carrier, include_cache=True) + with blosc2.RemoteCTable.open_reference(carrier, storage_options={"auto_mkdir": True}) as reopened: + before = reopened.traffic.requests + assert reopened["id"][5] == 5 + assert reopened.traffic.requests == before + + +def test_reference_retains_flattened_row_map(tmp_path): + path = tmp_path / "root.parquet" + carrier = tmp_path / "root.b2z" + source = pa.Table.from_arrays( + [pa.array([[{"id": i}] for i in range(6)], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + with blosc2.open(path, cache_dir=tmp_path / "cache") as remote: + assert remote["id"][5] == 5 + remote.save(carrier, include_cache=True) + with blosc2.open(carrier) as reopened: + assert len(reopened) == 6 + before = reopened.traffic.requests + assert reopened["id"][5] == 5 + assert reopened.traffic.requests == before + + +def test_reference_reader_options_are_frozen(tmp_path): + path = tmp_path / "source.parquet" + carrier = tmp_path / "source.b2z" + pq.write_table(pa.table({"name": ["a", "b", "a"]}), path) + with blosc2.open(path, parquet_options={"read_dictionary": ["name"]}) as remote: + assert remote["name"][0] == "a" + remote.save(carrier, include_cache=True) + with blosc2.RemoteCTable.open_reference(carrier) as reopened: + assert reopened.to_arrow().column("name").to_pylist() == ["a", "b", "a"] + with pytest.raises(TypeError, match="frozen"): + blosc2.RemoteCTable.open_reference(carrier, parquet_options={"read_dictionary": []}) + + +def test_closed_handle_rejects_cached_column(tmp_path): + path = tmp_path / "closed.parquet" + pq.write_table(pa.table({"a": [1, 2]}), path) + remote = blosc2.open(path) + column = remote._cols["a"] + remote.close() + with pytest.raises(RuntimeError, match="closed"): + column[0] + + +def test_refresh_invalidates_old_views(tmp_path): + path = tmp_path / "refresh.parquet" + pq.write_table(pa.table({"a": [1, 2]}), path) + with blosc2.open(path) as remote: + old_column = remote._cols["a"] + old_view = remote.select(["a"]) + pq.write_table(pa.table({"a": [3, 4]}), path) + remote.refresh() + assert remote["a"][:].tolist() == [3, 4] + with pytest.raises(RuntimeError, match=r"closed|stale"): + old_column[0] + with pytest.raises(RuntimeError, match=r"closed|stale"): + old_view.to_arrow() + + +def test_reader_options_and_eager_path(tmp_path): + path = tmp_path / "dictionary.parquet" + pq.write_table(pa.table({"name": ["aa", "bb", "aa", None]}), path, row_group_size=2) + options = {"read_dictionary": ["name"]} + with blosc2.CTable.from_parquet(path, **options) as eager: + with blosc2.open(path, parquet_options=options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + with blosc2.open(path, lazy=False, parquet_options=options) as imported: + assert imported.to_arrow().equals(eager.to_arrow()) + + +def test_timestamp_reader_coercion(tmp_path): + path = tmp_path / "int96.parquet" + source = pa.table({"time": pa.array([0, None, 1234], type=pa.timestamp("ms"))}) + pq.write_table(source, path, row_group_size=2, use_deprecated_int96_timestamps=True) + options = {"coerce_int96_timestamp_unit": "ms"} + with blosc2.CTable.from_parquet(path, **options) as eager: + with blosc2.open(path, parquet_options=options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_fixed_string_width_error_on_access(tmp_path): + path = tmp_path / "strings.parquet" + pq.write_table(pa.table({"s": ["a", "long"]}), path, row_group_size=1) + with blosc2.open(path, string_max_length=2) as remote: + assert remote["s"][0] == "a" + with pytest.raises(ValueError, match="longer than max_length=2"): + remote["s"][1] + + +def test_fixed_string_width_mapping_matches_eager(tmp_path): + path = tmp_path / "strings.parquet" + pq.write_table( + pa.table({"s": ["a", "é", None], "b": [b"\x00", b"hi", None], "v": ["wide", "text", ""]}), + path, + row_group_size=2, + ) + options = {"string_max_length": {"s": 2, "b": 2}} + with blosc2.CTable.from_parquet(path, **options) as eager, blosc2.open(path, **options) as remote: + assert schema_to_dict(remote._schema) == schema_to_dict(eager._schema) + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_reordered_duplicate_and_empty_rows(tmp_path): + path = tmp_path / "rows.parquet" + pq.write_table(pa.table({"x": [0, 1, 2, 3, 4]}), path, row_group_size=2) + with blosc2.open(path) as remote: + assert remote[-1].x == 4 + assert remote[[4, 1, 4]].to_arrow().column("x").to_pylist() == [4, 1, 4] + assert remote[2:0:-1].to_arrow().column("x").to_pylist() == [2, 1] + assert remote["x"][np.array([True, False, True, False, False])].tolist() == [0, 2] + assert len(remote[0:0]) == 0 + with pytest.raises(IndexError): + remote._cols["x"][5] + + +def test_dictionary_codes_change_between_groups(tmp_path): + path = tmp_path / "reordered-dictionary.parquet" + schema = pa.schema([pa.field("category", pa.dictionary(pa.int8(), pa.string()))]) + with pq.ParquetWriter(path, schema) as writer: + for dictionary in (["a", "b"], ["b", "a"]): + values = pa.DictionaryArray.from_arrays(pa.array([0, 1], type=pa.int8()), pa.array(dictionary)) + writer.write_batch(pa.record_batch([values], schema=schema)) + with blosc2.CTable.from_parquet(path) as eager, blosc2.open(path) as remote: + assert remote.to_arrow().equals(eager.to_arrow()) + + +def test_disk_cache_reuses_group_and_root_map(tmp_path): + path = tmp_path / "root.parquet" + source = pa.Table.from_arrays( + [pa.array([[{"id": i}] for i in range(6)], type=pa.list_(pa.struct([("id", pa.int32())])))], + names=[""], + ) + pq.write_table(source, path, row_group_size=2) + cache_dir = tmp_path / "cache" + with blosc2.open(path, cache_dir=cache_dir) as first: + prepared_requests = first.traffic.requests + assert first["id"][-1] == 5 + assert first.cache_bytes > 0 + with blosc2.open(path, cache_dir=cache_dir) as second: + reopened_requests = second.traffic.requests + assert reopened_requests < prepared_requests # row map reused + assert second["id"][-1] == 5 + assert second.traffic.requests == reopened_requests # converted group reused + cache_root = next(cache_dir.glob("root.parquet--*")) + generation = next(cache_root.glob("*.b2d")) + with blosc2.open(generation) as reopened: + assert reopened.col_names == ["id"] + before = reopened.traffic.requests + assert reopened["id"][-1] == 5 + assert reopened.traffic.requests == before + + +def test_warm_open_uses_retained_metadata_and_lazily_opens_arrow(tmp_path, monkeypatch): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir) as cold: + assert cold["x"][0] == 1 + + real_reader = pq.ParquetFile + readers = [] + + def record_reader(*args, **kwargs): + assert kwargs.get("metadata") is not None + readers.append(kwargs["metadata"]) + return real_reader(*args, **kwargs) + + monkeypatch.setattr(pq, "ParquetFile", record_reader) + real_from_arrow = CTable.from_arrow + monkeypatch.setattr(CTable, "from_arrow", lambda *args, **kwargs: pytest.fail("schema was inferred")) + monkeypatch.setattr(CTable, "load", lambda *args, **kwargs: pytest.fail("cached group was copied")) + monkeypatch.setattr(remote_parquet, "_source_marker", lambda *args: pytest.fail("source was checked")) + with blosc2.open(path, cache_dir=cache_dir) as warm: + assert warm.nrows == 4 + assert warm.metadata_bytes > 0 + assert readers == [] + assert warm["x"][0] == 1 + assert readers == [] + monkeypatch.setattr(CTable, "from_arrow", real_from_arrow) + assert warm["y"][3] == 8 + assert len(readers) == 1 + + +def test_retained_metadata_is_separate_for_projections(tmp_path): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2], "y": [3, 4]}), path) + with blosc2.open(path, cache_dir=cache_dir, columns=["x"]) as x_only: + assert x_only.col_names == ["x"] + with blosc2.open(path, cache_dir=cache_dir, columns=["y"]) as y_only: + assert y_only.col_names == ["y"] + assert len(list(cache_dir.glob("source.parquet--*"))) == 2 + + +def test_invalid_retained_metadata_is_rebuilt(tmp_path): + path = tmp_path / "source.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, cache_dir=cache_dir) as table: + disk = table._remote_storage()._owner.disk + manifest = disk.load() + manifest["metadata"]["parquet"]["discovery"]["schema"] = {"version": 999} + disk.publish(manifest) + with blosc2.open(path, cache_dir=cache_dir) as rebuilt: + assert rebuilt["x"][0] == 1 + assert ( + rebuilt._remote_storage()._owner.disk.load()["metadata"]["parquet"]["discovery"]["version"] == 1 + ) + + +@pytest.mark.parametrize("disk", [False, True]) +def test_cache_archive_reuses_warm_group_and_fetches_cold_group(tmp_path, disk, monkeypatch): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) + options = {"cache_dir": tmp_path / "cache"} if disk else {} + artifact = tmp_path / "reference.b2z" + cold_artifact = tmp_path / "cold-reference.b2z" + replace = os.replace + + def checked_replace(source, destination): + if destination in (artifact, cold_artifact): + assert source.parent.parent == destination.parent + return replace(source, destination) + + monkeypatch.setattr(remote_parquet.os, "replace", checked_replace) + with blosc2.open(path, **options) as original: + assert original["x"][0] == 1 + original.save(artifact) + original.save(cold_artifact, include_cache=False) + with blosc2.open(artifact) as restored: + assert restored.col_names == ["x", "y"] + before = restored.traffic.requests + assert restored["x"][0] == 1 + assert restored.traffic.requests == before + assert restored["y"][3] == 8 + assert restored.traffic.requests > before + with blosc2.open(cold_artifact) as cold: + before = cold.traffic.requests + assert cold["x"][0] == 1 + assert cold.traffic.requests > before + result = subprocess.check_output( + [sys.executable, "-c", "import blosc2,sys; print(blosc2.open(sys.argv[1])['x'][0])", str(artifact)], + text=True, + ) + assert result.strip() == "1" + + +def test_archive_seeds_shared_cache(tmp_path): + path = tmp_path / "source.parquet" + artifact = tmp_path / "reference.b2z" + pq.write_table(pa.table({"x": [1, 2, 3, 4]}), path, row_group_size=2) + with blosc2.open(path) as table: + assert table["x"][0] == 1 + table.save(artifact) + manifest, _ = blosc2.RemoteStore._load_artifact_manifest(str(artifact)) + conversion = dict(manifest["metadata"]["parquet"]["conversion"]) + conversion["_effective_null_policy"] = blosc2.NullPolicy( + **manifest["metadata"]["parquet"]["discovery"]["options"]["null_policy"] + ) + with blosc2.RemoteStore.with_sparse_cache( + path, + tmp_path / "shared", + carrier=artifact, + _parquet_conversion=conversion, + ) as store: + with store[""] as table: + before = store.traffic.requests + assert table["x"][0] == 1 + assert store.traffic.requests == before + + +def test_cache_generation_reopens_with_compression_options(tmp_path): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2, 3]}), path) + cache_dir = tmp_path / "cache" + with blosc2.open(path, cache_dir=cache_dir, cparams=blosc2.CParams(clevel=1)) as original: + assert original["x"][0] == 1 + generation = next(cache_dir.glob("*/*.b2d")) + with blosc2.open(generation) as reopened: + assert reopened["x"][0] == 1 + + +def test_shared_disk_cache_reuses_converted_group_across_processes(tmp_path): + path = tmp_path / "shared.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir, shared_cache=True) as remote: + assert remote["id"][5] == 5 + script = ( + "import blosc2,sys; " + "t=blosc2.open(sys.argv[1],cache_dir=sys.argv[2],shared_cache=True); " + "before=t.traffic.requests; print(t['id'][5],t.traffic.requests-before); t.close()" + ) + output = subprocess.check_output([sys.executable, "-c", script, str(path), str(cache_dir)], text=True) + assert output.strip() == "5 0" + with blosc2.RemoteCTable.with_sparse_cache(path, cache_dir) as remote: + before = remote.traffic.requests + assert remote["id"][5] == 5 + assert remote.traffic.requests == before + + +def test_shared_disk_cache_concurrent_cold_reads(tmp_path): + path = tmp_path / "shared-cold.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": list(range(12))}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir, shared_cache=True) as first: + with blosc2.open(path, cache_dir=cache_dir, shared_cache=True) as second: + baseline = first.traffic.requests + second.traffic.requests + with ThreadPoolExecutor(max_workers=2) as pool: + assert list(pool.map(lambda table: table["id"][11], (first, second))) == [11, 11] + assert first.traffic.requests + second.traffic.requests == baseline + 1 + + +def test_disk_cache_eviction_and_corrupt_entry(tmp_path): + path = tmp_path / "data.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": list(range(6))}), path, row_group_size=2) + with blosc2.open(path, cache_dir=cache_dir, max_cache_bytes=1) as remote: + assert remote["id"][2] == 2 + assert remote["id"][5] == 5 + entries = list(cache_dir.rglob("*.b2d")) + entries = [entry for entry in entries if entry.name.startswith(("0-", "1-", "2-"))] + assert not entries # The common cache budget does not retain an oversized group. + with blosc2.open(path, cache_dir=cache_dir, max_cache_bytes=None) as remote: + assert remote["id"][5] == 5 + entries = [entry for entry in cache_dir.rglob("*.b2d") if entry.name.startswith("2-")] + assert len(entries) == 1 + (entries[0] / "_meta.b2f").write_bytes(b"broken") + with blosc2.open(path, cache_dir=cache_dir) as remote: + assert remote["id"][5] == 5 + + +def test_refresh_disk_cache_uses_new_generation(tmp_path): + path = tmp_path / "changing.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": [1, 2]}), path) + with blosc2.open(path, cache_dir=cache_dir) as remote: + assert remote["id"][0] == 1 + pq.write_table(pa.table({"id": [3, 4, 5]}), path) + remote.refresh() + assert remote["id"][0] == 3 + + +def test_same_size_source_replacement_rebuilds_metadata(tmp_path): + path = tmp_path / "changing.parquet" + cache_dir = tmp_path / "cache" + pq.write_table(pa.table({"id": [1, 2]}), path) + old = path.stat() + with blosc2.open(path, cache_dir=cache_dir) as first: + assert first["id"][0] == 1 + pq.write_table(pa.table({"id": [3, 4]}), path) + assert path.stat().st_size == old.st_size + os.utime(path, ns=(old.st_atime_ns, old.st_mtime_ns + 2_000_000_000)) + with blosc2.open(path, cache_dir=cache_dir) as replaced: + assert replaced["id"][0] == 1 # Cached sources stay immutable until refresh. + replaced.refresh() + assert replaced["id"][0] == 3 + + +def test_refresh_unchanged_disk_source_keeps_cache_owner(tmp_path): + path = tmp_path / "unchanged.parquet" + pq.write_table(pa.table({"id": [1, 2]}), path) + with blosc2.open(path, cache_dir=tmp_path / "cache") as remote: + remote.refresh() + assert remote["id"][0] == 1 + + +def test_http_b2_file_id_supplies_missing_cache_validator(monkeypatch): + class Response: + def __init__(self): + self.headers = {"Content-Length": "49961641", "x-bz-file-id": "file-version-1"} + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + class Session: + def head(self, *args, **kwargs): + return Response() + + class Filesystem: + loop = fsspec.asyn.get_loop() + + def __init__(self): + self.kwargs = {} + + def info(self, path): + pytest.fail("HTTP source information required a second request") + + async def set_session(self): + return Session() + + def encode_url(self, path): + return path + + def _raise_not_found_for_status(self, response, path): + pass + + monkeypatch.setattr(fsspec.core, "url_to_fs", lambda *args, **kwargs: (Filesystem(), args[0])) + marker = remote_parquet._source_marker("https://example.com/table.parquet", None) + assert marker == {"size": "49961641", "x-bz-file-id": "file-version-1"} + + +def test_http_range_requests_are_narrow(tmp_path): + path = tmp_path / "served.parquet" + pq.write_table( + pa.table({"x": list(range(10_000)), "y": [f"value-{i:05d}" for i in range(10_000)]}), + path, + row_group_size=1_000, + ) + counts = {"requests": 0, "bytes": 0, "heads": 0} + + class Ranged(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + counts["heads"] += 1 + self.send_response(200) + self.send_header("Content-Length", str(path.stat().st_size)) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) + self.end_headers() + + def do_GET(self): + body = path.read_bytes() + span = self.headers.get("Range") + if span: + first, _, last = span.removeprefix("bytes=").partition("-") + if first: + first = int(first) + last = int(last) if last else len(body) - 1 + else: + first, last = max(0, len(body) - int(last)), len(body) - 1 + data = body[first : last + 1] + self.send_response(206) + self.send_header("Content-Range", f"bytes {first}-{last}/{len(body)}") + else: + data = body + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Last-Modified", formatdate(path.stat().st_mtime, usegmt=True)) + self.end_headers() + counts["requests"] += 1 + counts["bytes"] += len(data) + self.wfile.write(data) + + try: + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Ranged) + except PermissionError: + pytest.skip("localhost binding is blocked by the sandbox") + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}/served.parquet?signature=test" + with blosc2.open(url, storage_options={"block_size": 4096}) as remote: + assert counts["requests"] == 1 + assert counts["heads"] == 1 + assert "signature" not in remote.source["urlpath"] + with pytest.raises(ValueError, match="credential-like"): + remote.save(tmp_path / "signed.b2nd") + before = counts.copy() + assert remote["x"][9500] == 9500 + assert counts["requests"] > before["requests"] + assert counts["bytes"] - before["bytes"] < path.stat().st_size + warm = counts.copy() + assert remote["x"][9500] == 9500 + assert counts == warm + clean_url = url.split("?", 1)[0] + cache_dir = tmp_path / "http-cache" + with blosc2.open(clean_url, cache_dir=cache_dir) as remote: + assert remote["x"][9500] == 9500 + with blosc2.open(clean_url, cache_dir=cache_dir) as remote: + before = counts.copy() + assert remote["x"][9500] == 9500 + assert counts["requests"] == before["requests"] + finally: + server.shutdown() + server.server_close() + thread.join() + + +@pytest.mark.network +def test_s3_parquet_smoke(): + url = os.environ.get("BLOSC2_REMOTE_PARQUET_S3_URL") + if not url: + pytest.skip("set BLOSC2_REMOTE_PARQUET_S3_URL to an accessible Parquet file") + with blosc2.open(url, source_format="parquet") as remote: + assert remote.col_names + if len(remote): + remote[remote.col_names[0]][0] + + +def test_http_without_range_support_fails_clearly(tmp_path): + path = tmp_path / "large.parquet" + pq.write_table(pa.table({"x": list(range(20_000))}), path, row_group_size=1_000) + + class Unranged(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-Length", str(path.stat().st_size)) + self.end_headers() + + def do_GET(self): + body = path.read_bytes() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + try: + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Unranged) + except PermissionError: + pytest.skip("localhost binding is blocked by the sandbox") + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}/large.parquet" + with pytest.raises(ValueError, match="range requests"): + blosc2.open(url, storage_options={"block_size": 4096, "cache_type": "none"}) + finally: + server.shutdown() + server.server_close() + thread.join() + + +def test_shared_warm_read_holds_cache_lock(tmp_path, monkeypatch): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, cache_dir=tmp_path / "cache", shared_cache=True) as remote: + assert remote["x"][0] == 1 + owner = remote._remote_storage()._owner + guard = owner.disk.guard + locked = False + + @contextmanager + def tracked_guard(): + nonlocal locked + with guard(): + locked = True + try: + yield + finally: + locked = False + + monkeypatch.setattr(owner.disk, "guard", tracked_guard) + original_open = CTable.open + + def checked_open(*args, **kwargs): + table = original_open(*args, **kwargs) + column = table._cols["x"] + + class CheckedColumn: + def __getitem__(self, key): + assert locked, "shared cache must remain locked during lazy reads" + return column[key] + + table._cols["x"] = CheckedColumn() + return table + + monkeypatch.setattr(CTable, "open", checked_open) + assert remote["x"][0] == 1 + assert not locked + + +def test_archive_rejects_invalid_generation(tmp_path): + import zipfile + + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1]}), path) + archive_path = tmp_path / "reference.b2z" + with blosc2.open(path) as remote: + remote.save(archive_path) + with zipfile.ZipFile(archive_path) as archive: + carrier = blosc2.schunk_from_cframe(archive.read("embed.b2e")) + manifest = carrier.vlmeta["b2remote_manifest"] + manifest["generation"] = str(tmp_path / "escaped") + carrier = blosc2.SChunk(meta={"b2tree": {"version": 1}, "b2remote_store": {"version": 1}}) + carrier.vlmeta["b2remote_manifest"] = manifest + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("embed.b2e", carrier.to_cframe()) + with pytest.raises(ValueError, match="generation"): + blosc2.open(archive_path) + assert not (tmp_path / "escaped.b2d").exists() + + +@pytest.mark.parametrize("whole_row", [False, True]) +def test_refresh_waits_for_parquet_read(tmp_path, monkeypatch, whole_row): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]}), path, row_group_size=2) + with blosc2.open(path) as remote: + owner = remote._remote_storage()._owner + original_group = owner.group + entered = threading.Event() + resume = threading.Event() + refresh_started = threading.Event() + refresh_done = threading.Event() + + @contextmanager + def paused_group(number, physical): + with original_group(number, physical) as table: + if number == 0 and physical == "x": + entered.set() + assert resume.wait(5) + yield table + + monkeypatch.setattr(owner, "group", paused_group) + + def refresh(): + refresh_started.set() + remote.refresh() + refresh_done.set() + + with ThreadPoolExecutor(max_workers=2) as pool: + read = pool.submit(lambda: remote[0] if whole_row else remote["x"][:]) + assert entered.wait(5) + update = pool.submit(refresh) + assert refresh_started.wait(5) + assert not refresh_done.wait(0.1) + resume.set() + result = read.result(timeout=5) + update.result(timeout=5) + if whole_row: + assert (result.x, result.y) == (1, 5) + else: + assert result.tolist() == [1, 2, 3, 4] + + +def test_oversized_memory_group_is_not_retained(tmp_path): + path = tmp_path / "source.parquet" + pq.write_table(pa.table({"x": [1, 2]}), path) + with blosc2.open(path, max_cache_bytes=1) as remote: + assert remote["x"][0] == 1 + assert remote.cache_bytes == 0 + before = remote.traffic.requests + assert remote["x"][1] == 2 + assert remote.traffic.requests > before + + +@pytest.mark.parametrize(("workers", "budget", "expected"), [(1, 1 << 20, 1), (3, 1 << 20, 3), (3, 1, 1)]) +@pytest.mark.parametrize( + "policy", [blosc2.CachePolicy.MEMORY, blosc2.CachePolicy.NONE, blosc2.CachePolicy.DISK] +) +def test_parquet_parallel_reads_are_bounded(tmp_path, monkeypatch, workers, budget, expected, policy): + from blosc2.ctable_remote_read import column_values + + source = pa.table({name: [0, None, 2, 3, 4, 5] for name in ("x", "y", "z")}) + path = tmp_path / "parallel.parquet" + pq.write_table(source, path, row_group_size=3) + options = ( + {"cache_dir": tmp_path / "cache"} if policy is blosc2.CachePolicy.DISK else {"cache_policy": policy} + ) + with blosc2.open(path, max_concurrency=workers, **options) as remote: + remote.row_buffer_bytes = budget + owner = remote._remote_read_storage()._owner + fs = owner.parquet_handle.handle.fs + original = fs.cat_file + lock = threading.Lock() + barrier = threading.Barrier(expected) + state = {"active": 0, "peak": 0, "calls": 0, "bytes": 0} + caller = threading.current_thread() + group = remote_parquet.ParquetCache.group + + def checked_group(self, *args): + assert threading.current_thread() is caller + return group(self, *args) + + def fetch(*args, **kwargs): + with lock: + state["active"] += 1 + state["peak"] = max(state["peak"], state["active"]) + state["calls"] += 1 + first_wave = state["calls"] <= expected + try: + if first_wave: + barrier.wait(timeout=5) + data = original(*args, **kwargs) + with lock: + state["bytes"] += len(data) + return data + finally: + with lock: + state["active"] -= 1 + + monkeypatch.setattr(fs, "cat_file", fetch) + monkeypatch.setattr(remote_parquet.ParquetCache, "group", checked_group) + before = (remote.traffic.requests, remote.traffic.nbytes) + values = column_values(remote, remote.col_names, np.array([5, 1, 0, 5])) + assert values == {name: [5, None, 0, 5] for name in remote.col_names} + assert state["peak"] == expected + requests = 12 if policy is blosc2.CachePolicy.NONE else 6 + assert remote.traffic.requests - before[0] == state["calls"] == requests + assert remote.traffic.nbytes - before[1] == state["bytes"] + assert owner.parquet_handle.ranges == () + peak = remote._remote_read_storage()._peak_row_buffer_bytes + if budget > 1: + assert peak <= budget + else: + assert peak <= max( + owner.arrow_metadata.row_group(i).column(j).total_compressed_size + for i in range(2) + for j in range(3) + ) + warm = remote.traffic.requests + assert column_values(remote, remote.col_names, np.array([5, 1, 0, 5])) == values + assert remote.traffic.requests == warm + (requests if policy is blosc2.CachePolicy.NONE else 0) + + +def test_parquet_parallel_transport_failure_can_retry(tmp_path, monkeypatch): + from blosc2.ctable_remote_read import column_values + + path = tmp_path / "retry.parquet" + pq.write_table(pa.table({"x": [1, 2], "y": [3, 4]}), path) + with blosc2.open(path) as remote: + owner = remote._remote_read_storage()._owner + fs = owner.parquet_handle.handle.fs + original = fs.cat_file + + def fail(*args, **kwargs): + raise OSError("transport failed") + + monkeypatch.setattr(fs, "cat_file", fail) + with pytest.raises(OSError, match="transport failed"): + column_values(remote, remote.col_names, np.array([0])) + assert owner.parquet_handle.ranges == () + monkeypatch.setattr(fs, "cat_file", lambda *args, **kwargs: b"short") + with pytest.raises(OSError, match="Incomplete Parquet"): + column_values(remote, remote.col_names, np.array([0])) + monkeypatch.setattr(fs, "cat_file", original) + values = column_values(remote, remote.col_names, np.array([0])) + assert {name: list(value) for name, value in values.items()} == {"x": [1], "y": [3]} diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index e10b08279..da5589a8e 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -17,6 +17,7 @@ import subprocess import sys import threading +import time import numpy as np import pytest @@ -1091,6 +1092,9 @@ def test_cached_dir_refetches_on_same_size_change(tmp_path): localdir = blosc2.core.localize_fsspec_url("memory://samesize.b2d", tmp_path) assert pathlib.Path(localdir, "a.bin").read_bytes() == b"A" * 100 + # MemoryFileSystem's identity uses a creation timestamp; Windows clocks can + # return the same timestamp for consecutive writes. + time.sleep(0.05) memfs.pipe_file("/samesize.b2d/a.bin", b"B" * 100) localdir = blosc2.core.localize_fsspec_url("memory://samesize.b2d", tmp_path) assert pathlib.Path(localdir, "a.bin").read_bytes() == b"B" * 100 diff --git a/tests/test_remote_store.py b/tests/test_remote_store.py index 77d03423d..fd7265aa4 100644 --- a/tests/test_remote_store.py +++ b/tests/test_remote_store.py @@ -15,6 +15,96 @@ fsspec = pytest.importorskip("fsspec") +def test_parquet_table_root_uses_store_owner(tmp_path): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + path = tmp_path / "data.parquet" + pq.write_table(pa.table({"value": [1, 2, 3, 4]}), path, row_group_size=2) + with blosc2.RemoteStore(path, allow_table_root=True, _allow_local_source=True) as store: + assert store.kind("") == "ctable" + with store[""] as table: + assert table._storage._owner is store._owner + np.testing.assert_array_equal(table["value"][:], [1, 2, 3, 4]) + + +def test_parquet_shared_cache_export_and_refresh(tmp_path): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + path = tmp_path / "data.parquet" + pq.write_table(pa.table({"value": [1, 2, 3, 4]}), path, row_group_size=2) + url = "memory:///store-root.parquet" + fs = fsspec.filesystem("memory") + fs.pipe("/store-root.parquet", path.read_bytes()) + cache = tmp_path / "cache" + archive = tmp_path / "warm.b2z" + + def read(store): + with store[""] as table: + return table["value"][3] + + with blosc2.RemoteStore.with_sparse_cache(url, cache) as store: + table = store[""] + assert table._storage._owner is store._owner + assert store.read_cached_table(read) == (False, None) + assert table["value"][3] == 4 + assert store.read_cached_table(read) == (True, 4) + source = store._owner.disk.source + store.save(archive) + table.close() + + with blosc2.open(archive) as table: + before = table.traffic.requests + assert table["value"][3] == 4 + assert table.traffic.requests == before + + with blosc2.RemoteStore.with_sparse_cache(url, cache) as store: + before = store.traffic.requests + with store[""] as table: + assert table["value"][3] == 4 + assert store.traffic.requests == before + + removed, remaining = blosc2.RemoteStore.trim_sparse_cache(cache, source, 0) + assert removed + assert remaining == 0 + + with blosc2.RemoteStore.with_sparse_cache(url, cache) as store: + assert store.read_cached_table(read) == (False, None) + old = store[""] + pq.write_table(pa.table({"value": [5, 6, 7, 8]}), path, row_group_size=2) + fs.pipe("/store-root.parquet", path.read_bytes()) + store.refresh() + with pytest.raises(RuntimeError, match="stale"): + old["value"][3] + old.close() + with store[""] as table: + assert table["value"][3] == 8 + + +def test_parquet_standalone_refresh_replaces_cache(tmp_path, monkeypatch): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + path = tmp_path / "refresh.parquet" + pq.write_table(pa.table({"value": [1, 2]}), path) + with blosc2.RemoteCTable(path, cache_dir=tmp_path / "cache") as table: + old = table["value"] + assert old[0] == 1 + pq.write_table(pa.table({"value": [3, 4]}), path) + table.refresh() + with pytest.raises(RuntimeError, match=r"stale|closed"): + old[0] + assert table["value"][0] == 3 + import blosc2.remote_parquet as parquet + + def fail(*args): + raise OSError("discovery failed") + + with monkeypatch.context() as patch: + patch.setattr(parquet, "discover_parquet", fail) + with pytest.raises(OSError, match="discovery failed"): + table.refresh() + assert table["value"][0] == 3 + + @dataclasses.dataclass class NestedIndexedRow: value: int = blosc2.field(blosc2.int64(), chunks=(64,), blocks=(16,))