From a143a9123b8ed76cd3bac868ba710724a53becb8 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Mon, 5 Oct 2026 17:32:00 -0400 Subject: [PATCH 01/18] add tooks for remove Zarr access to notebook env --- notebooks/requirements_nb.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notebooks/requirements_nb.txt b/notebooks/requirements_nb.txt index c5abae8..baf1a88 100644 --- a/notebooks/requirements_nb.txt +++ b/notebooks/requirements_nb.txt @@ -1,2 +1,6 @@ # JupyterLab or equivalent is needed for the notebooks jupyterlab +anndata +dask +scanpy +aiohttp From 4e47bc1d6050415e212229b4c00f48df51d321b6 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Mon, 5 Oct 2026 17:32:55 -0400 Subject: [PATCH 02/18] build a FileSet for Zarr directories --- .../build_crate_from_dataset.py | 8 ++- src/crate_builder_script/croissant_wrapper.py | 52 +++++++++++++++++-- src/crate_builder_script/utils.py | 17 ++++++ 3 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 src/crate_builder_script/utils.py diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 05e42d5..e37c32b 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -4,6 +4,7 @@ import os from collections import defaultdict from datetime import datetime, timezone +from pathlib import Path from pprint import pformat from tempfile import TemporaryDirectory from typing import List @@ -14,7 +15,7 @@ fetch_entity_info, fetch_uuid_files_info, ) -from croissant_wrapper import CroissantWrapper +from croissant_wrapper import CroissantWrapper, detect_fileset_root from extractors import WrappedEntity from rocrate.model import ( ComputerLanguage, @@ -522,6 +523,7 @@ def main() -> None: blk_idx = {} for file_blk in uuid_files: blk_idx[file_blk["path"]] = file_blk + LOGGER.info(f"blk_idx is:\n{pformat(blk_idx, depth=1)}") # The dataset DOIs point to the Portal, which is basically a landing page, which is # forbidden as the direct link for a dataset under FAIR. So we can't use the DOI @@ -586,10 +588,12 @@ def main() -> None: if "files" in ds_entity: # This is a derived dataset- include only data products and qa_qc files for fl in ds_entity["files"]: + fileset_args = detect_fileset_root(fl["rel_path"]) if ( fl.get("is_data_product", False) or fl.get("is_qa_qc", False) or include_all_files + or fileset_args ): LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( @@ -598,6 +602,8 @@ def main() -> None: wrapped_croissant.add_file( ds_entity["uuid"], fl, blk_idx.get(fl["rel_path"]) ) + if fileset_args: + wrapped_croissant.add_fileset_from_root(fileset_args) else: LOGGER.debug(f"{fl['rel_path']} is not a data product or qa_qc") else: diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 6479bd0..f2e48b1 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -1,6 +1,7 @@ """Provides functions to build the Croissant file.""" import json import logging +import re from datetime import datetime, timezone from pathlib import Path from pprint import pformat @@ -8,9 +9,12 @@ import mlcroissant as mlc from api_calls import asset_url, entity_iri, fetch_entity_info from extractors import WrappedEntity +from utils import get_sha256_from_url LOGGER = logging.getLogger(__name__) +ANNDATA_ZARR_RE = re.compile(r"anndata-zarr/.*anndata.zarr/.zgroup") + EDAM_INFO = { ("EDAM_1.24.format_3727", None): {"desc": "tiff", "mime": "image/tiff"}, ("EDAM_1.24.format_3464", None): {"desc": "json", "mime": "application/json"}, @@ -42,6 +46,11 @@ "mime": "text/tab-separated-values", }, (None, ".csv"): {"desc": "unknown but csv encoded", "mime": "text/csv"}, + ("EDAM_1.24.format_2333", ""): { + # this entry covers the .zgroup entry at the top of a Zarr dir + "desc": "Zarr root file", + "mime": "application/json", + }, ("EDAM_1.24.format_2333", None): { "desc": "binary format", "mime": "application/octet-stream", @@ -53,6 +62,24 @@ } +def rel_path_to_id(rel_path: str) -> str: + return rel_path + + +def detect_fileset_root(rel_path: str) -> dict | None: + """If rel_path is a fileset root, return appropriate FileSet args""" + if ANNDATA_ZARR_RE.match(rel_path): + anchor_id = rel_path_to_id(rel_path) + return { + "id": f"anndata-zarr-dir-{anchor_id}", + "name": f"anndata_zarr_directory-{anchor_id}", + "contained_in": [anchor_id], + "includes": "../**/*", + "encoding_formats": ["application/x-zarr"] + } + return None + + def _ts_to_iso(ms): return ( datetime.fromtimestamp(ms / 1000, tz=timezone.utc).date().isoformat() @@ -301,19 +328,23 @@ def __init__(self, name: str, description: str): self.keywords = None def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None: - """Define and describe a single file in the dataset.""" + """ + Define and describe a single file in the dataset. + """ args = { - "id": file_info["rel_path"], + "id": rel_path_to_id(file_info["rel_path"]), "name": file_info["rel_path"], "description": file_info["description"], "content_url": asset_url(ds_uuid, file_info["rel_path"]), } + path_ext = Path(file_info["rel_path"]).suffix + LOGGER.info(f"EDAM lookup {file_info.get('edam_term')} <{path_ext}>") if edam := file_info.get("edam_term"): - if (edam, None) in EDAM_INFO: - args["encoding_formats"] = [EDAM_INFO[(edam, None)]["mime"]] - elif (edam, path_ext) in EDAM_INFO: + if (edam, path_ext) in EDAM_INFO: args["encoding_formats"] = [EDAM_INFO[(edam, path_ext)]["mime"]] + elif (edam, None) in EDAM_INFO: + args["encoding_formats"] = [EDAM_INFO[(edam, None)]["mime"]] elif (None, path_ext) in EDAM_INFO: args["encoding_formats"] = [EDAM_INFO[(None, path_ext)]["mime"]] else: @@ -323,8 +354,19 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None args["encoding_formats"] = ["application/octet-stream"] if file_blk: args["sha256"] = file_blk["sha256_checksum"] + else: + args["sha256"] = get_sha256_from_url(args["content_url"]) + LOGGER.info(f"HELLO {file_info['rel_path']} info:\n{pformat(args)}") + LOGGER.info(f"file_info is:\n{pformat(file_info)}") self.file_objects.append(mlc.FileObject(**args)) + + def add_fileset_from_root(self, fileset_args: dict) -> None: + """ + Add a fileset. + """ + self.file_objects.append(mlc.FileSet(**fileset_args)) + # def add_record_set(self, record_set: mlc.RecordSet): # self.record_sets.append(record_set) diff --git a/src/crate_builder_script/utils.py b/src/crate_builder_script/utils.py new file mode 100644 index 0000000..42e1a52 --- /dev/null +++ b/src/crate_builder_script/utils.py @@ -0,0 +1,17 @@ +"""provide small utility calls""" +import hashlib +import requests + +def get_sha256_from_url(url: str, chunk_size : int =16384) -> str: + """ + Given a file URL, return the sha256 of the file. + + Thanks to Google Gemini. Small files only, please. + """ + hash = hashlib.sha256() + with requests.get(url, stream=True) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=chunk_size): + if chunk: + hash.update(chunk) + return hash.hexdigest() From f75e90c7735bab9aa09121076f4e6c21a5702869 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 20 Oct 2026 18:02:43 -0400 Subject: [PATCH 03/18] produce a RecordSet from anndata-zarr --- .../build_crate_from_dataset.py | 10 +- src/crate_builder_script/croissant_wrapper.py | 98 ++++++++++++++++--- 2 files changed, 87 insertions(+), 21 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index e37c32b..c4f7c9d 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -15,7 +15,7 @@ fetch_entity_info, fetch_uuid_files_info, ) -from croissant_wrapper import CroissantWrapper, detect_fileset_root +from croissant_wrapper import CroissantWrapper, detect_recordset_root from extractors import WrappedEntity from rocrate.model import ( ComputerLanguage, @@ -588,12 +588,12 @@ def main() -> None: if "files" in ds_entity: # This is a derived dataset- include only data products and qa_qc files for fl in ds_entity["files"]: - fileset_args = detect_fileset_root(fl["rel_path"]) + recordset_args = detect_recordset_root(fl["rel_path"]) if ( fl.get("is_data_product", False) or fl.get("is_qa_qc", False) or include_all_files - or fileset_args + or recordset_args ): LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( @@ -602,8 +602,8 @@ def main() -> None: wrapped_croissant.add_file( ds_entity["uuid"], fl, blk_idx.get(fl["rel_path"]) ) - if fileset_args: - wrapped_croissant.add_fileset_from_root(fileset_args) + if recordset_args: + wrapped_croissant.add_recordset_from_root(recordset_args) else: LOGGER.debug(f"{fl['rel_path']} is not a data product or qa_qc") else: diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index f2e48b1..02aa4f7 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -1,4 +1,5 @@ """Provides functions to build the Croissant file.""" +import copy import json import logging import re @@ -13,7 +14,7 @@ LOGGER = logging.getLogger(__name__) -ANNDATA_ZARR_RE = re.compile(r"anndata-zarr/.*anndata.zarr/.zgroup") +ANNDATA_ZARR_RE = re.compile(r"(anndata-zarr/.*anndata.zarr)(/.zgroup)") EDAM_INFO = { ("EDAM_1.24.format_3727", None): {"desc": "tiff", "mime": "image/tiff"}, @@ -61,25 +62,79 @@ }, } +FILESET_KEYWORDS = ["id", "name", "contained_in", "includes", "encoding_formats"] -def rel_path_to_id(rel_path: str) -> str: - return rel_path +RECORDSET_KEYWORDS = ["id", "name", "description", "fields"] + +def _anndata_zarr_fs_func(fs_info: dict) -> dict: + rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} + LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") + return rslt + +def _anndata_zarr_rs_func(rs_info: dict) -> list[dict]: + id_base = rs_info["id"] + field_list = [ + { + "id": f"{id_base}/cells/cell-id", + "name": "cell_id", + "data_types": [mlc.DataType.TEXT], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.obs.ID") + ) + }, + { + "id": f"{id_base}/cells/expression-vector", + "name": "expression", + "data_types": [mlc.DataType.FLOAT], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.X") + ) + }, + { + "id": f"{id_base}/cells/cell-k-means-mean-expression", + "name": "cell-k-means-mean-expression", + "data_types": [mlc.DataType.FLOAT], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.obs.'Cell K-Means [Mean] Expression'") + ) + }, + ] + field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields + rslt = { + "id": f"{id_base}/cells-recordset", + "name": "cells-recordset", + "description": "individual cells data", + "fields": field_list + } + LOGGER.info(f"_anndata_zarr_rs_func returning:\n{pformat(rslt)}") + return [rslt] -def detect_fileset_root(rel_path: str) -> dict | None: - """If rel_path is a fileset root, return appropriate FileSet args""" - if ANNDATA_ZARR_RE.match(rel_path): - anchor_id = rel_path_to_id(rel_path) +def detect_recordset_root(rel_path: str) -> dict | None: + """If rel_path is a fileset root, return appropriate RecordSet info""" + if re_match := ANNDATA_ZARR_RE.match(rel_path): + # Information which directly describes the FileSet and can + # be used to derive the RecordSets + anchor_id = rel_path_to_id(re_match.group(1)) return { "id": f"anndata-zarr-dir-{anchor_id}", "name": f"anndata_zarr_directory-{anchor_id}", - "contained_in": [anchor_id], + "contained_in": [rel_path_to_id(rel_path)], "includes": "../**/*", - "encoding_formats": ["application/x-zarr"] + "encoding_formats": ["application/x-zarr"], + "fileset_func": _anndata_zarr_fs_func, + "recordset_func": _anndata_zarr_rs_func } return None +def rel_path_to_id(rel_path: str) -> str: + return rel_path + + def _ts_to_iso(ms): return ( datetime.fromtimestamp(ms / 1000, tz=timezone.utc).date().isoformat() @@ -360,15 +415,26 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None LOGGER.info(f"file_info is:\n{pformat(file_info)}") self.file_objects.append(mlc.FileObject(**args)) - - def add_fileset_from_root(self, fileset_args: dict) -> None: + def add_recordset_from_root(self, rs_info: dict) -> None: """ - Add a fileset. + Add one or more recordsets, and a fileset if necessary. + + rs_info is a dict returned by detect_recordset_root(). """ - self.file_objects.append(mlc.FileSet(**fileset_args)) - - # def add_record_set(self, record_set: mlc.RecordSet): - # self.record_sets.append(record_set) + if fileset_func := rs_info.get("fileset_func"): + self._add_fileset(fileset_func(rs_info)) + if recordset_func := rs_info.get("recordset_func"): + rs_info_list = recordset_func(rs_info) + for rs_info in rs_info_list: + self._add_recordset(rs_info) + + def _add_fileset(self, fs_info: dict) -> None: + """Add a FileSet using info returned by detect_recordset_root().""" + self.file_objects.append(mlc.FileSet(**fs_info)) + + def _add_recordset(self, rs_info: dict) -> None: + """Add a RecordSet.""" + self.record_sets.append(mlc.RecordSet(**rs_info)) def write(self, croissant_filename: str): """Write the Croissant file.""" From 58be7038c17bd585a8d60ec1ed5fb9baacf0d341 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 23 Oct 2026 11:55:50 -0400 Subject: [PATCH 04/18] add jsonpath-ng to requirements --- notebooks/requirements_nb.txt | 1 + src/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/notebooks/requirements_nb.txt b/notebooks/requirements_nb.txt index baf1a88..11b5075 100644 --- a/notebooks/requirements_nb.txt +++ b/notebooks/requirements_nb.txt @@ -4,3 +4,4 @@ anndata dask scanpy aiohttp +jsonpath-ng diff --git a/src/requirements.txt b/src/requirements.txt index f018eba..4600a96 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -7,3 +7,4 @@ bagit mlcroissant[dev]>=1.1.0 git+https://github.com/hubmapconsortium/cwltool.git@docker-gpu#egg=cwltool setuptools<82 +jsonpath-ng From d067f03dee8ebb5269d56ac97b0a07bb331ea2a6 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 23 Oct 2026 12:08:26 -0400 Subject: [PATCH 05/18] working on record generator for zarr. WIP --- .../build_crate_from_dataset.py | 2 +- src/crate_builder_script/croissant_wrapper.py | 25 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index c4f7c9d..25e1d81 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -588,7 +588,7 @@ def main() -> None: if "files" in ds_entity: # This is a derived dataset- include only data products and qa_qc files for fl in ds_entity["files"]: - recordset_args = detect_recordset_root(fl["rel_path"]) + recordset_args = detect_recordset_root(ds_entity["uuid"], fl["rel_path"]) if ( fl.get("is_data_product", False) or fl.get("is_qa_qc", False) diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 02aa4f7..4f323bd 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -8,6 +8,7 @@ from pprint import pformat import mlcroissant as mlc +import zarr from api_calls import asset_url, entity_iri, fetch_entity_info from extractors import WrappedEntity from utils import get_sha256_from_url @@ -73,6 +74,8 @@ def _anndata_zarr_fs_func(fs_info: dict) -> dict: def _anndata_zarr_rs_func(rs_info: dict) -> list[dict]: id_base = rs_info["id"] + obs_cols = rs_info.get("obs_columns") + print(f"POINT 3: obs_cols {obs_cols}") field_list = [ { "id": f"{id_base}/cells/cell-id", @@ -80,7 +83,7 @@ def _anndata_zarr_rs_func(rs_info: dict) -> list[dict]: "data_types": [mlc.DataType.TEXT], "source": mlc.Source( file_set=id_base, - extract=mlc.Extract(json_path="$.obs.ID") + extract=mlc.Extract(json_path="$.obs.index") ) }, { @@ -113,12 +116,25 @@ def _anndata_zarr_rs_func(rs_info: dict) -> list[dict]: return [rslt] -def detect_recordset_root(rel_path: str) -> dict | None: +def detect_recordset_root(ds_uuid: str, rel_path: str) -> dict | None: """If rel_path is a fileset root, return appropriate RecordSet info""" if re_match := ANNDATA_ZARR_RE.match(rel_path): # Information which directly describes the FileSet and can # be used to derive the RecordSets anchor_id = rel_path_to_id(re_match.group(1)) + zarr_url = asset_url(ds_uuid, anchor_id) + print(f"POINT 1: zarr_url = {zarr_url}") + import os + from anndata.io import read_elem + auth_tok = os.environ["AUTH_TOK"] + zarr_obs_store = zarr.open(zarr_url, mode="r", use_consolidated=False, + zarr_format=2, path="obs", + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) + zarr_elem = read_elem(zarr_obs_store) + print(f"POINT 1b: {zarr_elem.columns}") + print(f"POINT 1c: {list(zarr_elem.columns)}"), + obs_cols = list(zarr_elem.columns) + print(f"POINT 2: cols {obs_cols}") return { "id": f"anndata-zarr-dir-{anchor_id}", "name": f"anndata_zarr_directory-{anchor_id}", @@ -126,7 +142,8 @@ def detect_recordset_root(rel_path: str) -> dict | None: "includes": "../**/*", "encoding_formats": ["application/x-zarr"], "fileset_func": _anndata_zarr_fs_func, - "recordset_func": _anndata_zarr_rs_func + "recordset_func": _anndata_zarr_rs_func, + "obs_columns": obs_cols } return None @@ -411,7 +428,7 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None args["sha256"] = file_blk["sha256_checksum"] else: args["sha256"] = get_sha256_from_url(args["content_url"]) - LOGGER.info(f"HELLO {file_info['rel_path']} info:\n{pformat(args)}") + LOGGER.info(f"args for {file_info['rel_path']} info:\n{pformat(args)}") LOGGER.info(f"file_info is:\n{pformat(file_info)}") self.file_objects.append(mlc.FileObject(**args)) From 49dc23f90b87e0e06f1c9ac8228542bb46c3d4f6 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 23 Oct 2026 12:29:37 -0400 Subject: [PATCH 06/18] first pass separate record detectors. WIP --- .../anndata_zarr_tools.py | 101 ++++++++++++++++++ src/crate_builder_script/edam_tables.py | 46 ++++++++ 2 files changed, 147 insertions(+) create mode 100644 src/crate_builder_script/anndata_zarr_tools.py create mode 100644 src/crate_builder_script/edam_tables.py diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py new file mode 100644 index 0000000..ee08b2f --- /dev/null +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -0,0 +1,101 @@ +"""Provides functions to build the Croissant file.""" +import copy +import json +import logging +import re +from datetime import datetime, timezone +from pathlib import Path +from pprint import pformat + +import mlcroissant as mlc +import zarr +from api_calls import asset_url, entity_iri, fetch_entity_info +from extractors import WrappedEntity +from utils import get_sha256_from_url, FILESET_KEYWORDS, RECORDSET_KEYWORDS + +LOGGER = logging.getLogger(__name__) + +ANNDATA_ZARR_RE = re.compile(r"(anndata-zarr/.*anndata.zarr)(/.zgroup)") + +def anndata_zarr_fs_func(fs_info: dict) -> dict: + rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} + LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") + return rslt + +def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: + id_base = rs_info["id"] + obs_cols = rs_info.get("obs_columns") + print(f"POINT 3: obs_cols {obs_cols}") + field_list = [ + { + "id": f"{id_base}/cells/cell-id", + "name": "cell_id", + "data_types": [mlc.DataType.TEXT], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.obs.index") + ) + }, + { + "id": f"{id_base}/cells/expression-vector", + "name": "expression", + "data_types": [mlc.DataType.FLOAT], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.X") + ) + }, + { + "id": f"{id_base}/cells/cell-k-means-mean-expression", + "name": "cell-k-means-mean-expression", + "data_types": [mlc.DataType.FLOAT], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.obs.'Cell K-Means [Mean] Expression'") + ) + }, + ] + field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields + rslt = { + "id": f"{id_base}/cells-recordset", + "name": "cells-recordset", + "description": "individual cells data", + "fields": field_list + } + LOGGER.info(f"_anndata_zarr_rs_func returning:\n{pformat(rslt)}") + return [rslt] + + +def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None + """If rel_path is an anndata_zarr root, return appropriate RecordSet info""" + if re_match := ANNDATA_ZARR_RE.match(rel_path): + # Information which directly describes the FileSet and can + # be used to derive the RecordSets + anchor_id = rel_path_to_id(re_match.group(1)) + zarr_url = asset_url(ds_uuid, anchor_id) + print(f"POINT 1: zarr_url = {zarr_url}") + import os + from anndata.io import read_elem + auth_tok = os.environ["AUTH_TOK"] + zarr_obs_store = zarr.open(zarr_url, mode="r", use_consolidated=False, + zarr_format=2, path="obs", + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) + zarr_elem = read_elem(zarr_obs_store) + print(f"POINT 1b: {zarr_elem.columns}") + print(f"POINT 1c: {list(zarr_elem.columns)}"), + obs_cols = list(zarr_elem.columns) + print(f"POINT 2: cols {obs_cols}") + return { + "id": f"anndata-zarr-dir-{anchor_id}", + "name": f"anndata_zarr_directory-{anchor_id}", + "contained_in": [rel_path_to_id(rel_path)], + "includes": "../**/*", + "encoding_formats": ["application/x-zarr"], + "fileset_func": anndata_zarr_fs_func, + "recordset_func": anndata_zarr_rs_func, + "obs_columns": obs_cols + } + else: + return None + + diff --git a/src/crate_builder_script/edam_tables.py b/src/crate_builder_script/edam_tables.py new file mode 100644 index 0000000..46b37e2 --- /dev/null +++ b/src/crate_builder_script/edam_tables.py @@ -0,0 +1,46 @@ +"""Provides file type information based on edam code and extension.""" +EDAM_INFO = { + ("EDAM_1.24.format_3727", None): {"desc": "tiff", "mime": "image/tiff"}, + ("EDAM_1.24.format_3464", None): {"desc": "json", "mime": "application/json"}, + ("EDAM_1.24.format_3508", None): {"desc": "pdf", "mime": "application/pdf"}, + ("EDAM_1.24.format_3590", None): {"desc": "hdf5", "mime": "application/x-hdf5"}, + ("EDAM_1.24.format_3987", None): {"desc": "zip", "mime": "application/zip"}, + ("EDAM_1.24.format_3752", None): {"desc": "csv", "mime": "text/csv"}, + ("EDAM_1.24.format_3755", None): { + "desc": "tsv", + "mime": "text/tab-separated-values", + }, + ("EDAM_1.24.format_3790", None): { + "desc": "h5ad (anndata)", + "mime": "application/x-hdf5", + }, + ("EDAM_1.24.format_3915", None): {"desc": "zarr", "mime": "application/vnd.zarr"}, + ("EDAM_1.24.format_4006", None): { + "desc": "zarr (spatialdata)", + "mime": "application/vnd.zarr", + }, + ("EDAM_1.24.data_3671", None): {"desc": "plain text", "mime": "text/plain"}, + ("EDAM_1.24.format_3916", ".mtx"): { + "desc": "adjacency matrix", + "mime": "text/plain", + }, + (None, ".html"): {"desc": "unknown but html encoded", "mime": "text/html"}, + (None, ".tsv"): { + "desc": "unknown but tsv encoded", + "mime": "text/tab-separated-values", + }, + (None, ".csv"): {"desc": "unknown but csv encoded", "mime": "text/csv"}, + ("EDAM_1.24.format_2333", ""): { + # this entry covers the .zgroup entry at the top of a Zarr dir + "desc": "Zarr root file", + "mime": "application/json", + }, + ("EDAM_1.24.format_2333", None): { + "desc": "binary format", + "mime": "application/octet-stream", + }, + ("EDAM_1.24.format_3003", None): { + "desc": "BED (Browser Extensible Data)", + "mime": "text/x-bed", + }, +} \ No newline at end of file From 015fb1624d4ed605e8cebbc6702e01a0782a5c32 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 23 Oct 2026 12:34:32 -0400 Subject: [PATCH 07/18] separate anndata-zarr utils --- .../anndata_zarr_tools.py | 8 +- src/crate_builder_script/croissant_wrapper.py | 140 +----------------- src/crate_builder_script/utils.py | 9 ++ 3 files changed, 23 insertions(+), 134 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index ee08b2f..6960360 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -11,7 +11,11 @@ import zarr from api_calls import asset_url, entity_iri, fetch_entity_info from extractors import WrappedEntity -from utils import get_sha256_from_url, FILESET_KEYWORDS, RECORDSET_KEYWORDS +from utils import ( + get_sha256_from_url, + rel_path_to_id, + FILESET_KEYWORDS, RECORDSET_KEYWORDS +) LOGGER = logging.getLogger(__name__) @@ -66,7 +70,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: return [rslt] -def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None +def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: """If rel_path is an anndata_zarr root, return appropriate RecordSet info""" if re_match := ANNDATA_ZARR_RE.match(rel_path): # Information which directly describes the FileSet and can diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 4f323bd..c7ed1d4 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -11,147 +11,23 @@ import zarr from api_calls import asset_url, entity_iri, fetch_entity_info from extractors import WrappedEntity -from utils import get_sha256_from_url +from utils import ( + get_sha256_from_url, + rel_path_to_id, +) +from edam_tables import EDAM_INFO +from anndata_zarr_tools import anndata_zarr_detector LOGGER = logging.getLogger(__name__) -ANNDATA_ZARR_RE = re.compile(r"(anndata-zarr/.*anndata.zarr)(/.zgroup)") - -EDAM_INFO = { - ("EDAM_1.24.format_3727", None): {"desc": "tiff", "mime": "image/tiff"}, - ("EDAM_1.24.format_3464", None): {"desc": "json", "mime": "application/json"}, - ("EDAM_1.24.format_3508", None): {"desc": "pdf", "mime": "application/pdf"}, - ("EDAM_1.24.format_3590", None): {"desc": "hdf5", "mime": "application/x-hdf5"}, - ("EDAM_1.24.format_3987", None): {"desc": "zip", "mime": "application/zip"}, - ("EDAM_1.24.format_3752", None): {"desc": "csv", "mime": "text/csv"}, - ("EDAM_1.24.format_3755", None): { - "desc": "tsv", - "mime": "text/tab-separated-values", - }, - ("EDAM_1.24.format_3790", None): { - "desc": "h5ad (anndata)", - "mime": "application/x-hdf5", - }, - ("EDAM_1.24.format_3915", None): {"desc": "zarr", "mime": "application/vnd.zarr"}, - ("EDAM_1.24.format_4006", None): { - "desc": "zarr (spatialdata)", - "mime": "application/vnd.zarr", - }, - ("EDAM_1.24.data_3671", None): {"desc": "plain text", "mime": "text/plain"}, - ("EDAM_1.24.format_3916", ".mtx"): { - "desc": "adjacency matrix", - "mime": "text/plain", - }, - (None, ".html"): {"desc": "unknown but html encoded", "mime": "text/html"}, - (None, ".tsv"): { - "desc": "unknown but tsv encoded", - "mime": "text/tab-separated-values", - }, - (None, ".csv"): {"desc": "unknown but csv encoded", "mime": "text/csv"}, - ("EDAM_1.24.format_2333", ""): { - # this entry covers the .zgroup entry at the top of a Zarr dir - "desc": "Zarr root file", - "mime": "application/json", - }, - ("EDAM_1.24.format_2333", None): { - "desc": "binary format", - "mime": "application/octet-stream", - }, - ("EDAM_1.24.format_3003", None): { - "desc": "BED (Browser Extensible Data)", - "mime": "text/x-bed", - }, -} - -FILESET_KEYWORDS = ["id", "name", "contained_in", "includes", "encoding_formats"] - -RECORDSET_KEYWORDS = ["id", "name", "description", "fields"] - -def _anndata_zarr_fs_func(fs_info: dict) -> dict: - rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} - LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") - return rslt - -def _anndata_zarr_rs_func(rs_info: dict) -> list[dict]: - id_base = rs_info["id"] - obs_cols = rs_info.get("obs_columns") - print(f"POINT 3: obs_cols {obs_cols}") - field_list = [ - { - "id": f"{id_base}/cells/cell-id", - "name": "cell_id", - "data_types": [mlc.DataType.TEXT], - "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.obs.index") - ) - }, - { - "id": f"{id_base}/cells/expression-vector", - "name": "expression", - "data_types": [mlc.DataType.FLOAT], - "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.X") - ) - }, - { - "id": f"{id_base}/cells/cell-k-means-mean-expression", - "name": "cell-k-means-mean-expression", - "data_types": [mlc.DataType.FLOAT], - "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.obs.'Cell K-Means [Mean] Expression'") - ) - }, - ] - field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields - rslt = { - "id": f"{id_base}/cells-recordset", - "name": "cells-recordset", - "description": "individual cells data", - "fields": field_list - } - LOGGER.info(f"_anndata_zarr_rs_func returning:\n{pformat(rslt)}") - return [rslt] - def detect_recordset_root(ds_uuid: str, rel_path: str) -> dict | None: """If rel_path is a fileset root, return appropriate RecordSet info""" - if re_match := ANNDATA_ZARR_RE.match(rel_path): - # Information which directly describes the FileSet and can - # be used to derive the RecordSets - anchor_id = rel_path_to_id(re_match.group(1)) - zarr_url = asset_url(ds_uuid, anchor_id) - print(f"POINT 1: zarr_url = {zarr_url}") - import os - from anndata.io import read_elem - auth_tok = os.environ["AUTH_TOK"] - zarr_obs_store = zarr.open(zarr_url, mode="r", use_consolidated=False, - zarr_format=2, path="obs", - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) - zarr_elem = read_elem(zarr_obs_store) - print(f"POINT 1b: {zarr_elem.columns}") - print(f"POINT 1c: {list(zarr_elem.columns)}"), - obs_cols = list(zarr_elem.columns) - print(f"POINT 2: cols {obs_cols}") - return { - "id": f"anndata-zarr-dir-{anchor_id}", - "name": f"anndata_zarr_directory-{anchor_id}", - "contained_in": [rel_path_to_id(rel_path)], - "includes": "../**/*", - "encoding_formats": ["application/x-zarr"], - "fileset_func": _anndata_zarr_fs_func, - "recordset_func": _anndata_zarr_rs_func, - "obs_columns": obs_cols - } + if rslt := anndata_zarr_detector(ds_uuid, rel_path): + return rslt return None -def rel_path_to_id(rel_path: str) -> str: - return rel_path - - def _ts_to_iso(ms): return ( datetime.fromtimestamp(ms / 1000, tz=timezone.utc).date().isoformat() diff --git a/src/crate_builder_script/utils.py b/src/crate_builder_script/utils.py index 42e1a52..55f794b 100644 --- a/src/crate_builder_script/utils.py +++ b/src/crate_builder_script/utils.py @@ -2,6 +2,15 @@ import hashlib import requests +FILESET_KEYWORDS = ["id", "name", "contained_in", "includes", "encoding_formats"] + +RECORDSET_KEYWORDS = ["id", "name", "description", "fields"] + + +def rel_path_to_id(rel_path: str) -> str: + return rel_path + + def get_sha256_from_url(url: str, chunk_size : int =16384) -> str: """ Given a file URL, return the sha256 of the file. From d963a184ed75331fef2768b41f632113bbf325fc Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 27 Oct 2026 13:38:05 -0400 Subject: [PATCH 08/18] more Zarr parsing. WIP. --- .../anndata_zarr_tools.py | 97 +++++++++++++++---- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index 6960360..498e076 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -6,9 +6,13 @@ from datetime import datetime, timezone from pathlib import Path from pprint import pformat +import os +from anndata.io import read_elem import mlcroissant as mlc import zarr +import pandas as pd +import numpy as np from api_calls import asset_url, entity_iri, fetch_entity_info from extractors import WrappedEntity from utils import ( @@ -21,6 +25,14 @@ ANNDATA_ZARR_RE = re.compile(r"(anndata-zarr/.*anndata.zarr)(/.zgroup)") +PANDAS_TO_CROISSANT_TYPES = { + "int64": mlc.DataType.INTEGER, + "float64": mlc.DataType.FLOAT, + "bool": mlc.DataType.BOOL, + "object": mlc.DataType.TEXT, + "string": mlc.DataType.TEXT, +} + def anndata_zarr_fs_func(fs_info: dict) -> dict: rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") @@ -28,8 +40,14 @@ def anndata_zarr_fs_func(fs_info: dict) -> dict: def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: id_base = rs_info["id"] - obs_cols = rs_info.get("obs_columns") - print(f"POINT 3: obs_cols {obs_cols}") + obs_dtypes = rs_info.get("obs_dtypes", {}) + var_index = rs_info.get("var_index", {}) + X_shape = rs_info.get("X_shape", (-1, -1)) + print(f"TEST2a: {obs_dtypes}") + print(f"TEST2b: {var_index}") + print(f"TEST2c: {X_shape}") + + # These first two fields are always present field_list = [ { "id": f"{id_base}/cells/cell-id", @@ -44,21 +62,30 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: "id": f"{id_base}/cells/expression-vector", "name": "expression", "data_types": [mlc.DataType.FLOAT], + "is_array": True, + "array_shape": f"{X_shape[0]},{X_shape[1]}", "source": mlc.Source( file_set=id_base, extract=mlc.Extract(json_path="$.X") ) }, - { - "id": f"{id_base}/cells/cell-k-means-mean-expression", - "name": "cell-k-means-mean-expression", - "data_types": [mlc.DataType.FLOAT], - "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.obs.'Cell K-Means [Mean] Expression'") - ) - }, ] + + # Add a field for each column in obs + for col, dtype in obs_dtypes.items(): + if cr_dtype := PANDAS_TO_CROISSANT_TYPES.get(str(dtype)): + field_list.append({ + "id": f"{id_base}/cells/{col}", + "name": col, + "data_types": [cr_dtype], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path=f"$.obs.'{col}'") + ) + }) + else: + raise RuntimeError(f"zarr obs column {col} type {dtype} cannot be mapped to croissant") + field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields rslt = { "id": f"{id_base}/cells-recordset", @@ -78,17 +105,45 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: anchor_id = rel_path_to_id(re_match.group(1)) zarr_url = asset_url(ds_uuid, anchor_id) print(f"POINT 1: zarr_url = {zarr_url}") - import os - from anndata.io import read_elem auth_tok = os.environ["AUTH_TOK"] + # We can't actually open the X store because it's too large + # to work with here. But we need its row and column counts + # to specify the X array. We will need to get them from the + # obs and var stores. zarr_obs_store = zarr.open(zarr_url, mode="r", use_consolidated=False, - zarr_format=2, path="obs", + path="obs", storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) - zarr_elem = read_elem(zarr_obs_store) - print(f"POINT 1b: {zarr_elem.columns}") - print(f"POINT 1c: {list(zarr_elem.columns)}"), - obs_cols = list(zarr_elem.columns) - print(f"POINT 2: cols {obs_cols}") + zarr_obs_elem = read_elem(zarr_obs_store) + zarr_var_store = zarr.open(zarr_url, mode="r", use_consolidated=False, + path="var", + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) + zarr_var_elem = read_elem(zarr_var_store) + + # We have no examples of these, but they may be info we + # ultimately want to capture + for elt in ["varm", "varp", "obsm", "obsp", "uns"]: + store = zarr.open(zarr_url, mode="r", use_consolidated=False, + path=elt, + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) + elem = read_elem(store) + if isinstance(elem, pd.DataFrame): + print(f"TEST {elt} {type(elem)} {len(elem)}") + if len(elem): + LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" + " was detected but not translated") + for idx, row in elem.iterrows(): + print(f"{idx} {type(row)} {row}") + elif isinstance(elem, dict): + if elem: + LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" + " was detected but not translated") + else: + LOGGER.error(f"The AnnData Zarr element {elt} is unexpectedly" + f" of type {type(elem)}") + + obs_dtypes = zarr_obs_elem.dtypes + var_index = zarr_var_elem.index + print(f"TESTTEST: {zarr_obs_elem.shape[0]} {zarr_var_elem.shape[0]}") return { "id": f"anndata-zarr-dir-{anchor_id}", "name": f"anndata_zarr_directory-{anchor_id}", @@ -97,7 +152,9 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: "encoding_formats": ["application/x-zarr"], "fileset_func": anndata_zarr_fs_func, "recordset_func": anndata_zarr_rs_func, - "obs_columns": obs_cols + "obs_dtypes": obs_dtypes, + "var_index": var_index, + "X_shape": (zarr_obs_elem.shape[0], zarr_var_elem.shape[0]) } else: return None From 70ed4b2edbd6eb210a9759a83b630249a7d2a7a8 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 27 Oct 2026 16:21:20 -0400 Subject: [PATCH 09/18] working with better RecordSet structure --- .../anndata_zarr_tools.py | 116 ++++++++++++------ 1 file changed, 80 insertions(+), 36 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index 498e076..60b6b25 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -1,10 +1,6 @@ """Provides functions to build the Croissant file.""" -import copy -import json import logging import re -from datetime import datetime, timezone -from pathlib import Path from pprint import pformat import os @@ -12,13 +8,10 @@ import mlcroissant as mlc import zarr import pandas as pd -import numpy as np -from api_calls import asset_url, entity_iri, fetch_entity_info -from extractors import WrappedEntity +from api_calls import asset_url from utils import ( - get_sha256_from_url, rel_path_to_id, - FILESET_KEYWORDS, RECORDSET_KEYWORDS + FILESET_KEYWORDS ) LOGGER = logging.getLogger(__name__) @@ -43,24 +36,48 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: obs_dtypes = rs_info.get("obs_dtypes", {}) var_index = rs_info.get("var_index", {}) X_shape = rs_info.get("X_shape", (-1, -1)) - print(f"TEST2a: {obs_dtypes}") - print(f"TEST2b: {var_index}") - print(f"TEST2c: {X_shape}") - # These first two fields are always present + # Build the row index RecordSet + row_field = mlc.Field( + id=f"{id_base}/cells/cell-id", + name="cell_id", + data_types=[mlc.DataType.TEXT], + source=mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.obs.index") + ) + ) + field_list = [row_field] + row_index_rs = { + "id": f"{id_base}/cell-id-index", + "name": "cell-id-index", + "description": "individual cell ids", + "fields": field_list + } + + # Build the column index RecordSet + col_field = mlc.Field( + id=f"{id_base}/cells/channel-id", + name="channel_id", + data_types=[mlc.DataType.TEXT], + source=mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path="$.var.index") + ) + ) + field_list = [col_field] + col_index_rs = { + "id": f"{id_base}/channel-index", + "name": "cell-index", + "description": "individual channel ids", + "fields": field_list + } + + # Build the RecordSet for X, referencing the indices field_list = [ { - "id": f"{id_base}/cells/cell-id", - "name": "cell_id", - "data_types": [mlc.DataType.TEXT], - "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.obs.index") - ) - }, - { - "id": f"{id_base}/cells/expression-vector", - "name": "expression", + "id": f"{id_base}/cells/expression-matrix", + "name": "expression-matrix", "data_types": [mlc.DataType.FLOAT], "is_array": True, "array_shape": f"{X_shape[0]},{X_shape[1]}", @@ -69,9 +86,31 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: extract=mlc.Extract(json_path="$.X") ) }, + { + "id": f"{id_base}/cells/expression-rows", + "name": "expression-rows", + "description": "expression matrix row IDs", + "data_types": [mlc.DataType.TEXT], + "references": row_field + }, + { + "id": f"{id_base}/cells/expression-columns", + "name": "expresion-columns", + "description": "expression matrix column IDs", + "data_types": [mlc.DataType.TEXT], + "references": col_field + } ] + field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields + X_rs = { + "id": f"{id_base}/expression-matrix", + "name": "expression-matrix", + "description": "expression data matrix", + "fields": field_list + } - # Add a field for each column in obs + # Guild the Record of fields representing the obs data + field_list = [] for col, dtype in obs_dtypes.items(): if cr_dtype := PANDAS_TO_CROISSANT_TYPES.get(str(dtype)): field_list.append({ @@ -85,16 +124,26 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: }) else: raise RuntimeError(f"zarr obs column {col} type {dtype} cannot be mapped to croissant") - + field_list.append( + { + "id": f"{id_base}/cells/obs-rows", + "name": "obs-rows", + "description": "observation matrix row IDs", + "data_types": [mlc.DataType.TEXT], + "references": row_field + }, + ) field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields - rslt = { - "id": f"{id_base}/cells-recordset", - "name": "cells-recordset", - "description": "individual cells data", + obs_rs = { + "id": f"{id_base}/cells-obs", + "name": "cells-obs", + "description": "individual cells obs data", "fields": field_list } + + rslt = [row_index_rs, col_index_rs, X_rs, obs_rs] LOGGER.info(f"_anndata_zarr_rs_func returning:\n{pformat(rslt)}") - return [rslt] + return rslt def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: @@ -104,7 +153,6 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: # be used to derive the RecordSets anchor_id = rel_path_to_id(re_match.group(1)) zarr_url = asset_url(ds_uuid, anchor_id) - print(f"POINT 1: zarr_url = {zarr_url}") auth_tok = os.environ["AUTH_TOK"] # We can't actually open the X store because it's too large # to work with here. But we need its row and column counts @@ -127,12 +175,9 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) elem = read_elem(store) if isinstance(elem, pd.DataFrame): - print(f"TEST {elt} {type(elem)} {len(elem)}") if len(elem): LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" " was detected but not translated") - for idx, row in elem.iterrows(): - print(f"{idx} {type(row)} {row}") elif isinstance(elem, dict): if elem: LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" @@ -143,7 +188,6 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: obs_dtypes = zarr_obs_elem.dtypes var_index = zarr_var_elem.index - print(f"TESTTEST: {zarr_obs_elem.shape[0]} {zarr_var_elem.shape[0]}") return { "id": f"anndata-zarr-dir-{anchor_id}", "name": f"anndata_zarr_directory-{anchor_id}", From 075c053288d6350d32996d9c482b08506b7a80f8 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Wed, 28 Oct 2026 20:07:58 -0400 Subject: [PATCH 10/18] still trying to get remote zipped zarrs working. WIP. --- .../anndata_zarr_tools.py | 72 ++++++++++++++++--- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index 60b6b25..fd500b8 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -3,8 +3,9 @@ import re from pprint import pformat import os +import fsspec -from anndata.io import read_elem +from anndata.io import read_elem, read_zarr import mlcroissant as mlc import zarr import pandas as pd @@ -17,6 +18,7 @@ LOGGER = logging.getLogger(__name__) ANNDATA_ZARR_RE = re.compile(r"(anndata-zarr/.*anndata.zarr)(/.zgroup)") +ANNDATA_ZARR_ZIP_RE = re.compile(r"(anndata-zarr/.*anndata.zarr.zip)") PANDAS_TO_CROISSANT_TYPES = { "int64": mlc.DataType.INTEGER, @@ -148,24 +150,72 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: """If rel_path is an anndata_zarr root, return appropriate RecordSet info""" - if re_match := ANNDATA_ZARR_RE.match(rel_path): + re_match_fs = ANNDATA_ZARR_RE.match(rel_path) + re_match_zip = ANNDATA_ZARR_ZIP_RE.match(rel_path) + re_match_any = re_match_fs or re_match_zip + if re_match_any: # Information which directly describes the FileSet and can # be used to derive the RecordSets - anchor_id = rel_path_to_id(re_match.group(1)) + anchor_id = rel_path_to_id(re_match_any.group(1)) zarr_url = asset_url(ds_uuid, anchor_id) auth_tok = os.environ["AUTH_TOK"] # We can't actually open the X store because it's too large # to work with here. But we need its row and column counts # to specify the X array. We will need to get them from the # obs and var stores. - zarr_obs_store = zarr.open(zarr_url, mode="r", use_consolidated=False, - path="obs", - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) - zarr_obs_elem = read_elem(zarr_obs_store) - zarr_var_store = zarr.open(zarr_url, mode="r", use_consolidated=False, - path="var", - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) - zarr_var_elem = read_elem(zarr_var_store) + if re_match_fs: + zarr_obs_store = zarr.open( + zarr_url, mode="r", use_consolidated=False, + path="obs", + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} + ) + zarr_obs_elem = read_elem(zarr_obs_store) + zarr_var_store = zarr.open( + zarr_url, mode="r", use_consolidated=False, + path="var", + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} + ) + zarr_var_elem = read_elem(zarr_var_store) + elif re_match_zip: + print(f"ZARR URL {zarr_url}") + fs = fsspec.filesystem( + "zip", + fo=zarr_url, + mode="r", + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} + ) + all_files = fs.find("") + print("FILES") + for f in all_files[:15]: + print(f) + store_mapper = fs.get_mapper() + print(f"KEYS: {list(store_mapper.keys())}") + #root = zarr.open(store_mapper, mode="r", zarr_format=2) + store_obs_mapper = fs.get_mapper("obs") + print(f"POINT 0; obs keys: {list(store_obs_mapper.keys())}") + zarr_store = zarr.open_group( + store_mapper, + mode="r", + #zarr_format=2, + #use_consolidated=False + ) + + # zarr_obs_store = zarr.open_group( + # store_obs_mapper, + # mode="r", + # zarr_format=2, + # use_consolidated=False + # ) + print(f"POINT 1: {type(zarr_store)}") + #zarr_obs_elem = read_zarr(zarr_obs_store) + zarr_elem = read_zarr(zarr_store) + print("POINT 2") + #zarr_obs_elem = root["obs"] + #zarr_var_elem = root["var"] + else: + raise RuntimeError("Internal Error opening remote zarr") + print(f"zarr_obs_elem {zarr_obs_elem}") + print(f"zarr_var_elem {zarr_var_elem}") # We have no examples of these, but they may be info we # ultimately want to capture From 413928c7da2d3b80f303abc4f8da0b84b6af3458 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 29 Oct 2026 17:24:54 -0400 Subject: [PATCH 11/18] fix zarr to version 3 or above --- src/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/requirements.txt b/src/requirements.txt index 4600a96..28144e9 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -8,3 +8,4 @@ mlcroissant[dev]>=1.1.0 git+https://github.com/hubmapconsortium/cwltool.git@docker-gpu#egg=cwltool setuptools<82 jsonpath-ng +zarr>=3 From 4d0360240e06a3fb32d2704247e1f0450fc79aa1 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 29 Oct 2026 17:25:55 -0400 Subject: [PATCH 12/18] working support for remote zipped zarrs --- .../anndata_zarr_tools.py | 159 ++++++++++++------ 1 file changed, 105 insertions(+), 54 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index fd500b8..104a046 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -1,13 +1,18 @@ """Provides functions to build the Croissant file.""" import logging import re -from pprint import pformat +from pprint import pformat, pprint import os import fsspec +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.zip import ZipFileSystem +import zipfile +from collections.abc import MutableMapping from anndata.io import read_elem, read_zarr import mlcroissant as mlc import zarr +from zarr.core.buffer.core import default_buffer_prototype import pandas as pd from api_calls import asset_url from utils import ( @@ -22,17 +27,78 @@ PANDAS_TO_CROISSANT_TYPES = { "int64": mlc.DataType.INTEGER, + "uint8": mlc.DataType.INTEGER, "float64": mlc.DataType.FLOAT, "bool": mlc.DataType.BOOL, "object": mlc.DataType.TEXT, "string": mlc.DataType.TEXT, } + +class MapperWrapper(MutableMapping): + """Controls for bad behavior of AnnData 0.12 with Zarr for remote zips.""" + def __init__(self, mapper): + self.mapper = mapper + + def __getitem__(self, key): + print(f"GETITEM {key}") + return self.mapper[key] + + def __contains__(self, key): + return key in self.mapper + + def __keys__(self): + print("KEYS called") + return self.mapper.keys() + + def __setitem__(self, key, value): + self.mapper[key] = value + + def __delitem__(self, key): + del self.mapper[key] + + def __iter__(self): + print("ITER") + return iter(self.mapper) + + def __len__(self): + print("LEN") + return len(self.mapper) + + +# +# Copied from hubmapconsortium/portal-visualization utils.py +# many thanks to Nick Akhmetov +# +class SafeZipFileSystem(ZipFileSystem): + # zarr 3 treats FileNotFoundError as "key absent"; ZipFileSystem raises KeyError, + # which breaks zarr's probes for optional metadata files. + def _open(self, path, mode="rb", *args, **kwargs): + try: + return super()._open(path, mode, *args, **kwargs) + except KeyError as e: + raise FileNotFoundError(path) from e + + +def open_zip_zarr_store(url, token=None): + headers = {"Authorization": f"Bearer {token}"} if token else {} + fs = SafeZipFileSystem( + fo=url, + target_protocol="https", # NOT remote_protocol/remote_options: silently ignored + target_options={"client_kwargs": {"headers": headers}}, + ) + # zarr 3's FsspecStore needs an async filesystem; ZipFileSystem is sync. + store = zarr.storage.FsspecStore(AsyncFileSystemWrapper(fs, asynchronous=True), read_only=True) + pprint(dir(store)) + return store + + def anndata_zarr_fs_func(fs_info: dict) -> dict: rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") return rslt + def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: id_base = rs_info["id"] obs_dtypes = rs_info.get("obs_dtypes", {}) @@ -176,66 +242,51 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} ) zarr_var_elem = read_elem(zarr_var_store) + # We have no examples of these, but they may be info we + # ultimately want to capture + for elt in ["varm", "varp", "obsm", "obsp", "uns"]: + store = zarr.open(zarr_url, mode="r", use_consolidated=False, + path=elt, + storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) + elem = read_elem(store) + if isinstance(elem, pd.DataFrame): + if len(elem): + LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" + " was detected but not translated") + elif isinstance(elem, dict): + if elem: + LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" + " was detected but not translated") + else: + LOGGER.error(f"The AnnData Zarr element {elt} is unexpectedly" + f" of type {type(elem)}") elif re_match_zip: - print(f"ZARR URL {zarr_url}") - fs = fsspec.filesystem( - "zip", - fo=zarr_url, - mode="r", - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} - ) - all_files = fs.find("") - print("FILES") - for f in all_files[:15]: - print(f) - store_mapper = fs.get_mapper() - print(f"KEYS: {list(store_mapper.keys())}") - #root = zarr.open(store_mapper, mode="r", zarr_format=2) - store_obs_mapper = fs.get_mapper("obs") - print(f"POINT 0; obs keys: {list(store_obs_mapper.keys())}") - zarr_store = zarr.open_group( - store_mapper, - mode="r", - #zarr_format=2, - #use_consolidated=False - ) + zarr_store = open_zip_zarr_store(zarr_url, token=auth_tok) + zarr_group = zarr.open_group(zarr_store, mode="r", use_consolidated=False) + zarr_obs_elem = read_elem(zarr_group["obs"]) + zarr_var_elem = read_elem(zarr_group["var"]) + # We have no examples of these, but they may be info we + # ultimately want to capture + for elt in ["varm", "varp", "obsm", "obsp", "uns"]: + if elt in zarr_group: + elem = read_elem(zarr_group[elt]) + if isinstance(elem, pd.DataFrame): + if len(elem): + LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" + " was detected but not translated") + elif isinstance(elem, dict): + if elem: + LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" + " was detected but not translated") + else: + LOGGER.error(f"The AnnData Zarr element {elt} is unexpectedly" + f" of type {type(elem)}") - # zarr_obs_store = zarr.open_group( - # store_obs_mapper, - # mode="r", - # zarr_format=2, - # use_consolidated=False - # ) - print(f"POINT 1: {type(zarr_store)}") - #zarr_obs_elem = read_zarr(zarr_obs_store) - zarr_elem = read_zarr(zarr_store) - print("POINT 2") - #zarr_obs_elem = root["obs"] - #zarr_var_elem = root["var"] else: raise RuntimeError("Internal Error opening remote zarr") print(f"zarr_obs_elem {zarr_obs_elem}") print(f"zarr_var_elem {zarr_var_elem}") - # We have no examples of these, but they may be info we - # ultimately want to capture - for elt in ["varm", "varp", "obsm", "obsp", "uns"]: - store = zarr.open(zarr_url, mode="r", use_consolidated=False, - path=elt, - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) - elem = read_elem(store) - if isinstance(elem, pd.DataFrame): - if len(elem): - LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" - " was detected but not translated") - elif isinstance(elem, dict): - if elem: - LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" - " was detected but not translated") - else: - LOGGER.error(f"The AnnData Zarr element {elt} is unexpectedly" - f" of type {type(elem)}") - obs_dtypes = zarr_obs_elem.dtypes var_index = zarr_var_elem.index return { From 03da691639840080f9b84743fd84ef1c24cfc4d0 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 29 Oct 2026 18:00:59 -0400 Subject: [PATCH 13/18] cleanup; fix 'includes' entry --- .../anndata_zarr_tools.py | 113 ++++++------------ .../build_crate_from_dataset.py | 2 +- 2 files changed, 39 insertions(+), 76 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index 104a046..6c14964 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -1,19 +1,16 @@ """Provides functions to build the Croissant file.""" -import logging -import re -from pprint import pformat, pprint -import os -import fsspec from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.implementations.zip import ZipFileSystem -import zipfile -from collections.abc import MutableMapping +import logging +from pprint import pformat +import re +from typing import Any -from anndata.io import read_elem, read_zarr +from anndata.io import read_elem import mlcroissant as mlc -import zarr -from zarr.core.buffer.core import default_buffer_prototype import pandas as pd +import zarr + from api_calls import asset_url from utils import ( rel_path_to_id, @@ -35,44 +32,17 @@ } -class MapperWrapper(MutableMapping): - """Controls for bad behavior of AnnData 0.12 with Zarr for remote zips.""" - def __init__(self, mapper): - self.mapper = mapper - - def __getitem__(self, key): - print(f"GETITEM {key}") - return self.mapper[key] - - def __contains__(self, key): - return key in self.mapper - - def __keys__(self): - print("KEYS called") - return self.mapper.keys() - - def __setitem__(self, key, value): - self.mapper[key] = value - - def __delitem__(self, key): - del self.mapper[key] - - def __iter__(self): - print("ITER") - return iter(self.mapper) - - def __len__(self): - print("LEN") - return len(self.mapper) - - # # Copied from hubmapconsortium/portal-visualization utils.py # many thanks to Nick Akhmetov # class SafeZipFileSystem(ZipFileSystem): - # zarr 3 treats FileNotFoundError as "key absent"; ZipFileSystem raises KeyError, - # which breaks zarr's probes for optional metadata files. + """ + Define ZipFileSystem with better FileNoteFound behavior + + zarr 3 treats FileNotFoundError as "key absent"; ZipFileSystem raises KeyError, + which breaks zarr's probes for optional metadata files. + """ def _open(self, path, mode="rb", *args, **kwargs): try: return super()._open(path, mode, *args, **kwargs) @@ -81,6 +51,7 @@ def _open(self, path, mode="rb", *args, **kwargs): def open_zip_zarr_store(url, token=None): + """Open a remote zipped zarr file via URL.""" headers = {"Authorization": f"Bearer {token}"} if token else {} fs = SafeZipFileSystem( fo=url, @@ -94,12 +65,14 @@ def open_zip_zarr_store(url, token=None): def anndata_zarr_fs_func(fs_info: dict) -> dict: + """FileSet generator for anndata_zarr.""" rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") return rslt def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: + """RecordSet generator for anndata_zarr.""" id_base = rs_info["id"] obs_dtypes = rs_info.get("obs_dtypes", {}) var_index = rs_info.get("var_index", {}) @@ -214,6 +187,21 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: return rslt +def test_zarr_element_not_empty(key: str, elem: Any) -> None: + """Raise a warning/error if a zarr element is unexpectedly not empty""" + if isinstance(elem, pd.DataFrame): + if len(elem): + LOGGER.warning(f"AnnData Zarr element {key} (a Dataframe)" + " was detected but not translated") + elif isinstance(elem, dict): + if elem: + LOGGER.warning(f"AnnData Zarr element {key} (a dict)" + " was detected but not translated") + else: + LOGGER.error(f"The AnnData Zarr element {key} is unexpectedly" + f" of type {type(elem)}") + + def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: """If rel_path is an anndata_zarr root, return appropriate RecordSet info""" re_match_fs = ANNDATA_ZARR_RE.match(rel_path) @@ -225,10 +213,6 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: anchor_id = rel_path_to_id(re_match_any.group(1)) zarr_url = asset_url(ds_uuid, anchor_id) auth_tok = os.environ["AUTH_TOK"] - # We can't actually open the X store because it's too large - # to work with here. But we need its row and column counts - # to specify the X array. We will need to get them from the - # obs and var stores. if re_match_fs: zarr_obs_store = zarr.open( zarr_url, mode="r", use_consolidated=False, @@ -242,50 +226,29 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} ) zarr_var_elem = read_elem(zarr_var_store) - # We have no examples of these, but they may be info we + include_str = "../**/*" + # We have no examples of the following, but they may be info we # ultimately want to capture for elt in ["varm", "varp", "obsm", "obsp", "uns"]: store = zarr.open(zarr_url, mode="r", use_consolidated=False, path=elt, storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) elem = read_elem(store) - if isinstance(elem, pd.DataFrame): - if len(elem): - LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" - " was detected but not translated") - elif isinstance(elem, dict): - if elem: - LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" - " was detected but not translated") - else: - LOGGER.error(f"The AnnData Zarr element {elt} is unexpectedly" - f" of type {type(elem)}") + test_zarr_element_not_empty(elt, elem) elif re_match_zip: zarr_store = open_zip_zarr_store(zarr_url, token=auth_tok) zarr_group = zarr.open_group(zarr_store, mode="r", use_consolidated=False) zarr_obs_elem = read_elem(zarr_group["obs"]) zarr_var_elem = read_elem(zarr_group["var"]) - # We have no examples of these, but they may be info we + include_str = "*/*" + # We have no examples of the following, but they may be info we # ultimately want to capture for elt in ["varm", "varp", "obsm", "obsp", "uns"]: if elt in zarr_group: elem = read_elem(zarr_group[elt]) - if isinstance(elem, pd.DataFrame): - if len(elem): - LOGGER.warning(f"AnnData Zarr element {elt} (a Dataframe)" - " was detected but not translated") - elif isinstance(elem, dict): - if elem: - LOGGER.warning(f"AnnData Zarr element {elt} (a dict)" - " was detected but not translated") - else: - LOGGER.error(f"The AnnData Zarr element {elt} is unexpectedly" - f" of type {type(elem)}") - + test_zarr_element_not_empty(elt, elem) else: raise RuntimeError("Internal Error opening remote zarr") - print(f"zarr_obs_elem {zarr_obs_elem}") - print(f"zarr_var_elem {zarr_var_elem}") obs_dtypes = zarr_obs_elem.dtypes var_index = zarr_var_elem.index @@ -293,7 +256,7 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: "id": f"anndata-zarr-dir-{anchor_id}", "name": f"anndata_zarr_directory-{anchor_id}", "contained_in": [rel_path_to_id(rel_path)], - "includes": "../**/*", + "includes": include_str, "encoding_formats": ["application/x-zarr"], "fileset_func": anndata_zarr_fs_func, "recordset_func": anndata_zarr_rs_func, diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 25e1d81..312ac5f 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -63,13 +63,13 @@ # - the workflow_instance lacks start and end dates # - "homo sapiens" is hard-coded into the keywords. That will have to be # fixed for SenNet. -# - unpublished examples: # TARGET_ID = "HBM567.VCBK.562" # TARGET_ID = "HBM487.HJZB.546" # primary dataset # - published examples: # TARGET_ID = "HBM866.VMBK.952" # TARGET_ID = "HBM473.QLDT.264" # primary dataset # TARGET_ID = "HBM748.CHWC.963" # derived from snare-seq +# TARGET_ID = "HBM953.LMWQ.235" # CODEX processed with zipped Zarr ############### From b696a2048d12a26a2075b87388255b494ab70910 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 30 Oct 2026 12:35:45 -0400 Subject: [PATCH 14/18] lint, flake8, black --- .../anndata_zarr_tools.py | 134 ++++++++++-------- .../build_crate_from_dataset.py | 1 - src/crate_builder_script/croissant_wrapper.py | 15 +- src/crate_builder_script/edam_tables.py | 2 +- src/crate_builder_script/utils.py | 6 +- 5 files changed, 83 insertions(+), 75 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index 6c14964..0e8c90a 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -1,21 +1,18 @@ """Provides functions to build the Croissant file.""" -from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper -from fsspec.implementations.zip import ZipFileSystem import logging -from pprint import pformat +import os import re +from pprint import pformat from typing import Any -from anndata.io import read_elem import mlcroissant as mlc import pandas as pd import zarr - +from anndata.io import read_elem from api_calls import asset_url -from utils import ( - rel_path_to_id, - FILESET_KEYWORDS -) +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.zip import ZipFileSystem +from utils import FILESET_KEYWORDS, rel_path_to_id LOGGER = logging.getLogger(__name__) @@ -38,11 +35,12 @@ # class SafeZipFileSystem(ZipFileSystem): """ - Define ZipFileSystem with better FileNoteFound behavior + Define ZipFileSystem with better FileNoteFound behavior. zarr 3 treats FileNotFoundError as "key absent"; ZipFileSystem raises KeyError, which breaks zarr's probes for optional metadata files. """ + def _open(self, path, mode="rb", *args, **kwargs): try: return super()._open(path, mode, *args, **kwargs) @@ -59,23 +57,23 @@ def open_zip_zarr_store(url, token=None): target_options={"client_kwargs": {"headers": headers}}, ) # zarr 3's FsspecStore needs an async filesystem; ZipFileSystem is sync. - store = zarr.storage.FsspecStore(AsyncFileSystemWrapper(fs, asynchronous=True), read_only=True) - pprint(dir(store)) + store = zarr.storage.FsspecStore( + AsyncFileSystemWrapper(fs, asynchronous=True), read_only=True + ) return store def anndata_zarr_fs_func(fs_info: dict) -> dict: - """FileSet generator for anndata_zarr.""" + """Generate a FileSet for anndata_zarr.""" rslt = {key: fs_info[key] for key in FILESET_KEYWORDS} LOGGER.info(f"_anndata_zarr_fs_func returning:\n{pformat(rslt)}") return rslt def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: - """RecordSet generator for anndata_zarr.""" + """Generate a RecordSet for anndata_zarr.""" id_base = rs_info["id"] obs_dtypes = rs_info.get("obs_dtypes", {}) - var_index = rs_info.get("var_index", {}) X_shape = rs_info.get("X_shape", (-1, -1)) # Build the row index RecordSet @@ -84,16 +82,15 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: name="cell_id", data_types=[mlc.DataType.TEXT], source=mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.obs.index") - ) + file_set=id_base, extract=mlc.Extract(json_path="$.obs.index") + ), ) field_list = [row_field] row_index_rs = { "id": f"{id_base}/cell-id-index", "name": "cell-id-index", "description": "individual cell ids", - "fields": field_list + "fields": field_list, } # Build the column index RecordSet @@ -102,16 +99,15 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: name="channel_id", data_types=[mlc.DataType.TEXT], source=mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.var.index") - ) + file_set=id_base, extract=mlc.Extract(json_path="$.var.index") + ), ) field_list = [col_field] col_index_rs = { "id": f"{id_base}/channel-index", "name": "cell-index", "description": "individual channel ids", - "fields": field_list + "fields": field_list, } # Build the RecordSet for X, referencing the indices @@ -123,55 +119,58 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: "is_array": True, "array_shape": f"{X_shape[0]},{X_shape[1]}", "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path="$.X") - ) + file_set=id_base, extract=mlc.Extract(json_path="$.X") + ), }, { "id": f"{id_base}/cells/expression-rows", "name": "expression-rows", "description": "expression matrix row IDs", "data_types": [mlc.DataType.TEXT], - "references": row_field + "references": row_field, }, { "id": f"{id_base}/cells/expression-columns", "name": "expresion-columns", "description": "expression matrix column IDs", "data_types": [mlc.DataType.TEXT], - "references": col_field - } + "references": col_field, + }, ] field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields X_rs = { "id": f"{id_base}/expression-matrix", "name": "expression-matrix", "description": "expression data matrix", - "fields": field_list + "fields": field_list, } # Guild the Record of fields representing the obs data field_list = [] for col, dtype in obs_dtypes.items(): if cr_dtype := PANDAS_TO_CROISSANT_TYPES.get(str(dtype)): - field_list.append({ - "id": f"{id_base}/cells/{col}", - "name": col, - "data_types": [cr_dtype], - "source": mlc.Source( - file_set=id_base, - extract=mlc.Extract(json_path=f"$.obs.'{col}'") - ) - }) + field_list.append( + { + "id": f"{id_base}/cells/{col}", + "name": col, + "data_types": [cr_dtype], + "source": mlc.Source( + file_set=id_base, + extract=mlc.Extract(json_path=f"$.obs.'{col}'"), + ), + } + ) else: - raise RuntimeError(f"zarr obs column {col} type {dtype} cannot be mapped to croissant") + raise RuntimeError( + f"zarr obs column {col} type {dtype} cannot be mapped to croissant" + ) field_list.append( { "id": f"{id_base}/cells/obs-rows", "name": "obs-rows", "description": "observation matrix row IDs", "data_types": [mlc.DataType.TEXT], - "references": row_field + "references": row_field, }, ) field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields @@ -179,7 +178,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: "id": f"{id_base}/cells-obs", "name": "cells-obs", "description": "individual cells obs data", - "fields": field_list + "fields": field_list, } rslt = [row_index_rs, col_index_rs, X_rs, obs_rs] @@ -188,22 +187,27 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: def test_zarr_element_not_empty(key: str, elem: Any) -> None: - """Raise a warning/error if a zarr element is unexpectedly not empty""" + """Raise a warning/error if a zarr element is unexpectedly not empty.""" if isinstance(elem, pd.DataFrame): if len(elem): - LOGGER.warning(f"AnnData Zarr element {key} (a Dataframe)" - " was detected but not translated") + LOGGER.warning( + f"AnnData Zarr element {key} (a Dataframe)" + " was detected but not translated" + ) elif isinstance(elem, dict): if elem: - LOGGER.warning(f"AnnData Zarr element {key} (a dict)" - " was detected but not translated") + LOGGER.warning( + f"AnnData Zarr element {key} (a dict)" + " was detected but not translated" + ) else: - LOGGER.error(f"The AnnData Zarr element {key} is unexpectedly" - f" of type {type(elem)}") + LOGGER.error( + f"The AnnData Zarr element {key} is unexpectedly" f" of type {type(elem)}" + ) -def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: - """If rel_path is an anndata_zarr root, return appropriate RecordSet info""" +def anndata_zarr_detector(ds_uuid: str, rel_path: str) -> dict | None: + """If rel_path is an anndata_zarr root, return appropriate RecordSet info.""" re_match_fs = ANNDATA_ZARR_RE.match(rel_path) re_match_zip = ANNDATA_ZARR_ZIP_RE.match(rel_path) re_match_any = re_match_fs or re_match_zip @@ -215,24 +219,34 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: auth_tok = os.environ["AUTH_TOK"] if re_match_fs: zarr_obs_store = zarr.open( - zarr_url, mode="r", use_consolidated=False, + zarr_url, + mode="r", + use_consolidated=False, path="obs", - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} + storage_options={"headers": {"Authorization": f"Bearer {auth_tok}"}}, ) zarr_obs_elem = read_elem(zarr_obs_store) zarr_var_store = zarr.open( - zarr_url, mode="r", use_consolidated=False, + zarr_url, + mode="r", + use_consolidated=False, path="var", - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}} + storage_options={"headers": {"Authorization": f"Bearer {auth_tok}"}}, ) zarr_var_elem = read_elem(zarr_var_store) include_str = "../**/*" # We have no examples of the following, but they may be info we # ultimately want to capture for elt in ["varm", "varp", "obsm", "obsp", "uns"]: - store = zarr.open(zarr_url, mode="r", use_consolidated=False, - path=elt, - storage_options={"headers":{"Authorization": f"Bearer {auth_tok}"}}) + store = zarr.open( + zarr_url, + mode="r", + use_consolidated=False, + path=elt, + storage_options={ + "headers": {"Authorization": f"Bearer {auth_tok}"} + }, + ) elem = read_elem(store) test_zarr_element_not_empty(elt, elem) elif re_match_zip: @@ -262,9 +276,7 @@ def anndata_zarr_detector(ds_uuid: str, rel_path:str) -> dict | None: "recordset_func": anndata_zarr_rs_func, "obs_dtypes": obs_dtypes, "var_index": var_index, - "X_shape": (zarr_obs_elem.shape[0], zarr_var_elem.shape[0]) + "X_shape": (zarr_obs_elem.shape[0], zarr_var_elem.shape[0]), } else: return None - - diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 312ac5f..f57702d 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -4,7 +4,6 @@ import os from collections import defaultdict from datetime import datetime, timezone -from pathlib import Path from pprint import pformat from tempfile import TemporaryDirectory from typing import List diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index c7ed1d4..01fbcc9 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -1,28 +1,25 @@ """Provides functions to build the Croissant file.""" -import copy import json import logging -import re from datetime import datetime, timezone from pathlib import Path from pprint import pformat import mlcroissant as mlc -import zarr +from anndata_zarr_tools import anndata_zarr_detector from api_calls import asset_url, entity_iri, fetch_entity_info +from edam_tables import EDAM_INFO from extractors import WrappedEntity from utils import ( get_sha256_from_url, rel_path_to_id, ) -from edam_tables import EDAM_INFO -from anndata_zarr_tools import anndata_zarr_detector LOGGER = logging.getLogger(__name__) def detect_recordset_root(ds_uuid: str, rel_path: str) -> dict | None: - """If rel_path is a fileset root, return appropriate RecordSet info""" + """If rel_path is a fileset root, return appropriate RecordSet info.""" if rslt := anndata_zarr_detector(ds_uuid, rel_path): return rslt return None @@ -276,9 +273,7 @@ def __init__(self, name: str, description: str): self.keywords = None def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None: - """ - Define and describe a single file in the dataset. - """ + """Define and describe a single file in the dataset.""" args = { "id": rel_path_to_id(file_info["rel_path"]), "name": file_info["rel_path"], @@ -311,7 +306,7 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None def add_recordset_from_root(self, rs_info: dict) -> None: """ Add one or more recordsets, and a fileset if necessary. - + rs_info is a dict returned by detect_recordset_root(). """ if fileset_func := rs_info.get("fileset_func"): diff --git a/src/crate_builder_script/edam_tables.py b/src/crate_builder_script/edam_tables.py index 46b37e2..e3692b1 100644 --- a/src/crate_builder_script/edam_tables.py +++ b/src/crate_builder_script/edam_tables.py @@ -43,4 +43,4 @@ "desc": "BED (Browser Extensible Data)", "mime": "text/x-bed", }, -} \ No newline at end of file +} diff --git a/src/crate_builder_script/utils.py b/src/crate_builder_script/utils.py index 55f794b..2e43e76 100644 --- a/src/crate_builder_script/utils.py +++ b/src/crate_builder_script/utils.py @@ -1,5 +1,6 @@ -"""provide small utility calls""" +"""Provide small utility calls.""" import hashlib + import requests FILESET_KEYWORDS = ["id", "name", "contained_in", "includes", "encoding_formats"] @@ -8,10 +9,11 @@ def rel_path_to_id(rel_path: str) -> str: + """Provide a standard mapping from paths to ID strings.""" return rel_path -def get_sha256_from_url(url: str, chunk_size : int =16384) -> str: +def get_sha256_from_url(url: str, chunk_size: int = 16384) -> str: """ Given a file URL, return the sha256 of the file. From 322c1728b2251b82f2336b822bfe17e88d85ef0a Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 30 Oct 2026 12:49:29 -0400 Subject: [PATCH 15/18] bump zarr to > 3 in notebook requirements --- notebooks/requirements_nb.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/requirements_nb.txt b/notebooks/requirements_nb.txt index 11b5075..9e2695e 100644 --- a/notebooks/requirements_nb.txt +++ b/notebooks/requirements_nb.txt @@ -1,6 +1,7 @@ # JupyterLab or equivalent is needed for the notebooks jupyterlab anndata +zarr<3 dask scanpy aiohttp From 2e1a612c6a851d2140784175b17ce26fcf086d6a Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 30 Oct 2026 23:04:48 -0400 Subject: [PATCH 16/18] introduce utils.safe_id --- .../anndata_zarr_tools.py | 26 +++++++++---------- src/crate_builder_script/utils.py | 12 ++++++++- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index 0e8c90a..d65d212 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -12,7 +12,7 @@ from api_calls import asset_url from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.implementations.zip import ZipFileSystem -from utils import FILESET_KEYWORDS, rel_path_to_id +from utils import FILESET_KEYWORDS, rel_path_to_id, safe_id LOGGER = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: # Build the row index RecordSet row_field = mlc.Field( - id=f"{id_base}/cells/cell-id", + id=safe_id(id_base, "/cells/cell-id"), name="cell_id", data_types=[mlc.DataType.TEXT], source=mlc.Source( @@ -87,7 +87,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: ) field_list = [row_field] row_index_rs = { - "id": f"{id_base}/cell-id-index", + "id": safe_id(id_base, "/cell-id-index"), "name": "cell-id-index", "description": "individual cell ids", "fields": field_list, @@ -95,7 +95,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: # Build the column index RecordSet col_field = mlc.Field( - id=f"{id_base}/cells/channel-id", + id=safe_id(id_base, "/cells/channel-id"), name="channel_id", data_types=[mlc.DataType.TEXT], source=mlc.Source( @@ -104,7 +104,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: ) field_list = [col_field] col_index_rs = { - "id": f"{id_base}/channel-index", + "id": safe_id(id_base, "/channel-index"), "name": "cell-index", "description": "individual channel ids", "fields": field_list, @@ -113,7 +113,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: # Build the RecordSet for X, referencing the indices field_list = [ { - "id": f"{id_base}/cells/expression-matrix", + "id": safe_id(id_base, "/cells/expression-matrix"), "name": "expression-matrix", "data_types": [mlc.DataType.FLOAT], "is_array": True, @@ -123,14 +123,14 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: ), }, { - "id": f"{id_base}/cells/expression-rows", + "id": safe_id(id_base, "/cells/expression-rows"), "name": "expression-rows", "description": "expression matrix row IDs", "data_types": [mlc.DataType.TEXT], "references": row_field, }, { - "id": f"{id_base}/cells/expression-columns", + "id": safe_id(id_base, "/cells/expression-columns"), "name": "expresion-columns", "description": "expression matrix column IDs", "data_types": [mlc.DataType.TEXT], @@ -139,7 +139,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: ] field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields X_rs = { - "id": f"{id_base}/expression-matrix", + "id": safe_id(id_base, "/expression-matrix"), "name": "expression-matrix", "description": "expression data matrix", "fields": field_list, @@ -151,7 +151,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: if cr_dtype := PANDAS_TO_CROISSANT_TYPES.get(str(dtype)): field_list.append( { - "id": f"{id_base}/cells/{col}", + "id": safe_id(id_base, f"/cells/{col}"), "name": col, "data_types": [cr_dtype], "source": mlc.Source( @@ -166,7 +166,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: ) field_list.append( { - "id": f"{id_base}/cells/obs-rows", + "id": safe_id(id_base, "/cells/obs-rows"), "name": "obs-rows", "description": "observation matrix row IDs", "data_types": [mlc.DataType.TEXT], @@ -175,7 +175,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: ) field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields obs_rs = { - "id": f"{id_base}/cells-obs", + "id": safe_id(id_base, "/cells-obs"), "name": "cells-obs", "description": "individual cells obs data", "fields": field_list, @@ -267,7 +267,7 @@ def anndata_zarr_detector(ds_uuid: str, rel_path: str) -> dict | None: obs_dtypes = zarr_obs_elem.dtypes var_index = zarr_var_elem.index return { - "id": f"anndata-zarr-dir-{anchor_id}", + "id": safe_id(f"anndata-zarr-dir-{anchor_id}"), "name": f"anndata_zarr_directory-{anchor_id}", "contained_in": [rel_path_to_id(rel_path)], "includes": include_str, diff --git a/src/crate_builder_script/utils.py b/src/crate_builder_script/utils.py index 2e43e76..32bae68 100644 --- a/src/crate_builder_script/utils.py +++ b/src/crate_builder_script/utils.py @@ -8,9 +8,19 @@ RECORDSET_KEYWORDS = ["id", "name", "description", "fields"] +def safe_id(base: str, key: str | None = None) -> str: + """Work around mlcommons restrictions on id strings.""" + if key: + id = f"{base}/{key}" + else: + id = f"{base}" + id = id.replace(" ","--") + return id + + def rel_path_to_id(rel_path: str) -> str: """Provide a standard mapping from paths to ID strings.""" - return rel_path + return safe_id(rel_path) def get_sha256_from_url(url: str, chunk_size: int = 16384) -> str: From 25c29cb8c0e876c401454c1817d91f2ec2f74f3b Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 3 Nov 2026 11:37:00 -0500 Subject: [PATCH 17/18] fix reference-building format --- src/crate_builder_script/anndata_zarr_tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crate_builder_script/anndata_zarr_tools.py b/src/crate_builder_script/anndata_zarr_tools.py index d65d212..5b5ee0f 100644 --- a/src/crate_builder_script/anndata_zarr_tools.py +++ b/src/crate_builder_script/anndata_zarr_tools.py @@ -127,14 +127,14 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: "name": "expression-rows", "description": "expression matrix row IDs", "data_types": [mlc.DataType.TEXT], - "references": row_field, + "references": mlc.Source(field=row_field.id), }, { "id": safe_id(id_base, "/cells/expression-columns"), "name": "expresion-columns", "description": "expression matrix column IDs", "data_types": [mlc.DataType.TEXT], - "references": col_field, + "references": mlc.Source(field=col_field.id), }, ] field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields @@ -170,7 +170,7 @@ def anndata_zarr_rs_func(rs_info: dict) -> list[dict]: "name": "obs-rows", "description": "observation matrix row IDs", "data_types": [mlc.DataType.TEXT], - "references": row_field, + "references": mlc.Source(field=row_field.id), }, ) field_list = [mlc.Field(**elt) for elt in field_list] # convert to fields From 9d2526b3a3aca88f0fd063b60153b44a41e96ad3 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 3 Nov 2026 11:55:06 -0500 Subject: [PATCH 18/18] black again --- src/crate_builder_script/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate_builder_script/utils.py b/src/crate_builder_script/utils.py index 32bae68..4b21d5b 100644 --- a/src/crate_builder_script/utils.py +++ b/src/crate_builder_script/utils.py @@ -14,7 +14,7 @@ def safe_id(base: str, key: str | None = None) -> str: id = f"{base}/{key}" else: id = f"{base}" - id = id.replace(" ","--") + id = id.replace(" ", "--") return id