From 973d7f9f59d9ef9f063940f2b4c226662d6dae6c Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:10:05 -0500 Subject: [PATCH 01/25] Rewrite the base Importer class to allow more generic file reading and more generic file associations --- sasdata/data_io/importer.py | 216 ++++++++++++++++++++++++++++++------ 1 file changed, 183 insertions(+), 33 deletions(-) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index 4a80d4ab1..c732bb5c8 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -1,57 +1,207 @@ -import mimetypes +""" +Base class and content-based dispatch for all sasdata importers. + +Every module in :mod:`sasdata.data_io.importers` defines a child of :class:`Importer`. Each child knows + +* how to recognise a file it can read, from the file *contents* (:meth:`Importer.can_import`), and +* how to read that file into SasMeasurement objects (:meth:`Importer.load_data`). + +Children register themselves automatically when they are defined. :meth:`Importer.find_importer` then picks the +appropriate child for any file, so callers do not need to know the file format (or trust the file extension):: + + from sasdata.data_io.importer import Importer + + data = Importer.load("my_file.xml") # dict[str, SasMeasurement] + measurements, errors = Importer().import_data(["a.h5", "b.ses", "c.txt"]) +""" + +import importlib +import io from itertools import zip_longest from pathlib import Path +from typing import IO, ClassVar from sasdata.data import SasMeasurement -from sasdata.data_io.importers import * Path_Type = str | Path +Source_Type = Path_Type | IO[bytes] Value_Type = str | int | float | list[str] | list[int] | list[float] +# Number of bytes read from the start of a file when sniffing its contents. +SNIFF_SIZE = 4096 + class Importer: - def __init__(self): - pass + """ + Base class for all data importers. + + Child classes must implement :meth:`load_data` and :meth:`can_import`, and may set :attr:`priority` (lower values + are tested first) and :attr:`name`. Instantiating ``Importer`` directly gives a generic importer whose + :meth:`load_data` and :meth:`import_data` select the correct child importer for each file automatically. + """ + + # Human-readable name of the format handled by the importer. + name: ClassVar[str] = "Generic" + # Order in which the importer is tested when inferring the file type; lower values are tested first. + # Generic fallback importers (e.g. ASCII) should use a high value. + priority: ClassVar[int] = 100 + # All concrete importers, populated automatically by :meth:`__init_subclass__`. + _registry: ClassVar[list[type["Importer"]]] = [] + + def __init_subclass__(cls, register: bool = True, **kwargs): + super().__init_subclass__(**kwargs) + if register and cls not in Importer._registry: + Importer._registry.append(cls) + + def __init__(self, config: dict[str, Value_Type] | None = None): + self.config: dict[str, Value_Type] = config if config is not None else {} + + # ------------------------------------------------------------------------------------------------------------------ + # Interface to be implemented by every child importer + # ------------------------------------------------------------------------------------------------------------------ + + @classmethod + def can_import(cls, source: Source_Type) -> bool: + """Return True if the contents of ``source`` look like a file this importer can read. Must not raise.""" + return False + + def load_data(self, source: Source_Type) -> dict[str, SasMeasurement]: + """ + Load all measurements contained in ``source``. + + :param source: Path to the file to load (or an open binary file object, where supported). + :return: A dictionary of the measurements found in the file, keyed by a name unique within the file. + """ + importer = self.find_importer(source) + if importer is None: + raise ValueError(f'Unable to determine the file type of "{source}".') + return importer(self.config).load_data(source) + + # ------------------------------------------------------------------------------------------------------------------ + # File-type inference + # ------------------------------------------------------------------------------------------------------------------ + + @staticmethod + def importers() -> list[type["Importer"]]: + """All registered importers, in the order in which they will be tested.""" + # Importing the package imports every importer module, which registers the child classes. + importlib.import_module("sasdata.data_io.importers") + return sorted(Importer._registry, key=lambda importer: importer.priority) + + @classmethod + def find_importer(cls, source: Source_Type) -> type["Importer"] | None: + """Infer the type of ``source`` from its contents and return the importer class able to read it.""" + for importer in cls.importers(): + if importer.can_import(source): + return importer + return None + + @classmethod + def load(cls, source: Source_Type, config: dict[str, Value_Type] | None = None) -> dict[str, SasMeasurement]: + """Convenience method: infer the file type of ``source`` and load it.""" + return Importer(config).load_data(source) + + # ------------------------------------------------------------------------------------------------------------------ + # Bulk import + # ------------------------------------------------------------------------------------------------------------------ def import_data(self, files: list[Path_Type], configs: list[dict[str, Value_Type]] | None = None) \ - -> (list[SasMeasurement], list[str]): - output = [] - errors = [] + -> tuple[list[SasMeasurement], list[str]]: + """ + Import a series of files, inferring the importer for each one from its contents. + + :param files: Files to import. + :param configs: Optional per-file configuration. If there are fewer configs than files, the last config is + reused for the remaining files. + :return: All of the measurements loaded, and a list of error messages for anything that could not be loaded. + """ + output: list[SasMeasurement] = [] + errors: list[str] = [] if configs is None: configs = [] fill_value = configs[-1] if len(configs) > 0 else None for file, config in zip_longest(files, configs, fillvalue=fill_value): - file = Path(file) - measurements, errors = self._import_from_url(file, config) + if file is None: + break + measurements, file_errors = self._import_file(Path(file), config) output.extend(measurements) - errors.extend(errors) + errors.extend(file_errors) return output, errors - def _import_from_url(self, url: Path_Type, config: dict[str, Value_Type] | None = None) -> (list[SasMeasurement], list[str]): - file = Path(url) + def _import_file(self, file: Path, config: dict[str, Value_Type] | None = None) \ + -> tuple[list[SasMeasurement], list[str]]: if not file.exists(): return [], [f'File does not exist: "{file}"'] - # Deprecation Warning: - # mimetypes.guess_type() is soft deprecated in favor of mimetypes.guess_file_type() as of python 3.13 - # Once we move forward from python 3.12, this line should change - mime_type, encoding = mimetypes.guess_type(url) - imported = [] - errors = [] - # TODO: Need a viable config system that can pass a dictionary of value locations to each importer + importer = self.find_importer(file) + if importer is None: + return [], [f'Unable to determine the file type of "{file}"'] + merged_config = {**self.config, **(config or {})} try: - match mime_type: - case 'application/xml' | 'text/xml': - file_dict = import_xml.load_data(file) - imported.extend(list(file_dict.values())) - case 'application/vnd.hdfgroup.hdf5': - file_list = import_hdf5.load_data(file) - imported.extend(file_list) - case _: - file_list = import_ascii.load_data_default_params(file) - imported.extend(file_list) + imported = list(importer(merged_config).load_data(file).values()) except Exception as e: - errors.append(f'Error accessing "{file}": {e}') - for file in imported: - if hasattr(file, "errors"): - errors.extend(file.errors) + return [], [f'Error loading "{file}" with the {importer.name} importer: {e}'] + errors = [] + for measurement in imported: + errors.extend(getattr(measurement, "errors", [])) return imported, errors + + +# ---------------------------------------------------------------------------------------------------------------------- +# Helpers shared by the importers when sniffing file contents +# ---------------------------------------------------------------------------------------------------------------------- + +def read_header(source: Source_Type, size: int = SNIFF_SIZE) -> bytes: + """ + Read up to ``size`` bytes from the start of ``source`` without changing the position of a file object. + Returns an empty bytes object if the source cannot be read. + """ + try: + if isinstance(source, (str, Path)): + path = Path(source) + if not path.is_file(): + return b"" + with open(path, "rb") as f: + return f.read(size) + if isinstance(source, io.IOBase) or hasattr(source, "read"): + position = source.tell() + try: + source.seek(0) + header = source.read(size) + finally: + source.seek(position) + return header if isinstance(header, bytes) else header.encode() + except (OSError, ValueError): + pass + return b"" + + +def is_text(header: bytes) -> bool: + """Heuristic check that a file header is text rather than binary data.""" + if not header or b"\x00" in header: + return False + try: + header.decode("utf-8") + except UnicodeDecodeError as e: + # The header may have been truncated in the middle of a multibyte character. + if e.start < len(header) - 4: + try: + header.decode("latin-1") + except UnicodeDecodeError: + return False + # latin-1 decodes anything, so reject if there are many control characters. + controls = sum(1 for b in header if b < 32 and b not in (9, 10, 12, 13)) + return controls / len(header) < 0.05 + return True + + +def measurements_by_name(measurements: list[SasMeasurement]) -> dict[str, SasMeasurement]: + """Key a list of measurements by name, adding a numeric suffix to any repeated names so none are lost.""" + output: dict[str, SasMeasurement] = {} + for measurement in measurements: + key = measurement.name + index = 2 + while key in output: + key = f"{measurement.name} ({index})" + index += 1 + output[key] = measurement + return output From a938366bdea62953b5bcdfb4b05c55817fe12211 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:11:50 -0500 Subject: [PATCH 02/25] Convert the ascii reader to an Importer object that auto-associates itself in the generic Importer class --- sasdata/data_io/importers/import_ascii.py | 50 +++++++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index 713e4de0e..b467bc191 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -13,6 +13,7 @@ pairings, ) from sasdata.data import SasMeasurement +from sasdata.data_io.importer import Importer, Source_Type, is_text, measurements_by_name, read_header from sasdata.dataset_types import DatasetType, one_dim, unit_kinds from sasdata.default_units import get_default_unit from sasdata.guess import ( @@ -186,7 +187,7 @@ def merge_uncertainties(quantities: dict[str, Quantity]) -> dict[str, Quantity]: return new_quantities -def load_data(params: AsciiReaderParams) -> list[SasMeasurement]: +def load_measurements(params: AsciiReaderParams) -> list[SasMeasurement]: """ This loads a series of SasMeasurement objects based on the params. The amount of SasMeasurement objects loaded will depend on how many @@ -218,6 +219,47 @@ def load_data(params: AsciiReaderParams) -> list[SasMeasurement]: return loaded_data -def load_data_default_params(filename: str | Path) -> list[SasMeasurement]: - params = guess_params_from_filename(filename, guess_dataset_type(filename)) - return load_data(params) +class AsciiImporter(Importer): + """ + Importer for column-based ASCII files. + + The importer can be given an :class:`AsciiReaderParams` object (e.g. generated by the ASCII Reader Dialog in + SasView) describing how to read the file(s). If no parameters are given, they are guessed from the file contents. + As almost any text file can be read as ASCII, this importer is the fallback used when no other importer recognises + a file, so it is tested last. + """ + + name = "ASCII" + priority = 1000 + + def __init__(self, config: dict | None = None, params: AsciiReaderParams | None = None): + super().__init__(config) + self.params = params + + @classmethod + def can_import(cls, source: Source_Type) -> bool: + """Any text file on disk could be a column-based ASCII file.""" + if not isinstance(source, (str, Path)): + return False + return is_text(read_header(source)) + + def load_data(self, source: Source_Type | None = None) -> dict[str, SasMeasurement]: + """ + Load ASCII data, keyed by the basename of each file. + + :param source: The file to load. If parameters were given to the importer, they are used to read this file; + otherwise, the parameters are guessed from the file contents. If ``source`` is None, every file listed in + the importer's parameters is loaded. + """ + if source is None: + if self.params is None: + raise ValueError("A file to load, or AsciiReaderParams listing the files to load, must be given.") + params = self.params + elif not isinstance(source, (str, Path)): + raise TypeError("The ASCII importer can only load files from a path.") + elif self.params is not None: + params = replace(self.params, filenames=[str(source)]) + else: + filename = str(source) + params = guess_params_from_filename(filename, guess_dataset_type(filename)) + return measurements_by_name(load_measurements(params)) From 613b7dfa88d1e0070b3d5f8f81f9b86a29af5efb Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:13:14 -0500 Subject: [PATCH 03/25] Import public h5py classes and not ones from private classes --- sasdata/data_io/importers/import_hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sasdata/data_io/importers/import_hdf5.py b/sasdata/data_io/importers/import_hdf5.py index 373ebde17..a5d3ff8dc 100644 --- a/sasdata/data_io/importers/import_hdf5.py +++ b/sasdata/data_io/importers/import_hdf5.py @@ -4,8 +4,8 @@ import h5py import numpy as np -from h5py._hl.dataset import Dataset as HDF5Dataset -from h5py._hl.group import Group as HDF5Group +from h5py import Dataset as HDF5Dataset +from h5py import Group as HDF5Group from sasdata.data import SasMeasurement from sasdata.data_backing import Dataset as SASDataDataset From 2587af14bf87df6d25782fd38819b84cba2885fe Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:14:20 -0500 Subject: [PATCH 04/25] Convert hdf5 importer to a child of the Importer class --- sasdata/data_io/importers/import_hdf5.py | 106 +++++++++++++++-------- 1 file changed, 68 insertions(+), 38 deletions(-) diff --git a/sasdata/data_io/importers/import_hdf5.py b/sasdata/data_io/importers/import_hdf5.py index a5d3ff8dc..6fb1ccf46 100644 --- a/sasdata/data_io/importers/import_hdf5.py +++ b/sasdata/data_io/importers/import_hdf5.py @@ -10,6 +10,7 @@ from sasdata.data import SasMeasurement from sasdata.data_backing import Dataset as SASDataDataset from sasdata.data_backing import Group as SASDataGroup +from sasdata.data_io.importer import Importer, Source_Type, read_header from sasdata.dataset_types import one_dim, three_dim, two_dim from sasdata.metadata import ( Aperture, @@ -406,54 +407,83 @@ def parse_metadata(node : HDF5Group) -> Metadata: raw=raw) -def load_data(filename: str | Path) -> dict[str, SasMeasurement]: - with h5py.File(filename, "r") as f: - loaded_data: dict[str, SasMeasurement] = {} +# The HDF5 format signature. It is normally at the start of the file, but may be offset by a user block whose size is +# 512 bytes or a larger power of two. +HDF5_SIGNATURE = b"\x89HDF\r\n\x1a\n" - for root_key in f.keys(): - entry = f[root_key] - # if this is actually a SASentry - if not get_canSAS_class(entry) == 'SASentry': - continue - data_contents : dict[str, Quantity] = {} +class HDF5Importer(Importer): + """Importer for NXcanSAS (and older SasView canSAS) HDF5 files.""" - entry_keys = entry.keys() + name = "NXcanSAS HDF5" + priority = 10 - if not [k for k in entry_keys if get_canSAS_class(entry[k])=='SASdata']: - logger.warning("No sasdata or data key") - logger.warning(f"Known keys: {[k for k in entry_keys]}") - - metadata = parse_metadata(f[root_key]) - - for key in entry_keys: - component = entry[key] - if get_canSAS_class(entry[key])=='SASdata': - datum = recurse_hdf5(component) - data_contents = connected_data(datum, str(filename), metadata) - - if "Qz" in data_contents: - dataset_type = three_dim - elif "Qy" in data_contents: - dataset_type = two_dim - else: - dataset_type = one_dim + @classmethod + def can_import(cls, source: Source_Type) -> bool: + """The file is HDF5 if the HDF5 format signature is found at offset 0 or at a user block boundary.""" + if isinstance(source, (str, Path)): + try: + return h5py.is_hdf5(source) + except Exception: + return False + header = read_header(source, size=4096) + offset = 0 + while offset + len(HDF5_SIGNATURE) <= len(header): + if header[offset:offset + len(HDF5_SIGNATURE)] == HDF5_SIGNATURE: + return True + offset = 512 if offset == 0 else offset * 2 + return False + + def load_data(self, source: Source_Type) -> dict[str, SasMeasurement]: + """Load all SASentry groups in an HDF5 file""" + filename = source + with h5py.File(filename, "r") as f: + loaded_data: dict[str, SasMeasurement] = {} + + for root_key in f.keys(): + entry = f[root_key] + + # if this is actually a SASentry + if not get_canSAS_class(entry) == 'SASentry': + continue + data_contents : dict[str, Quantity] = {} + + entry_keys = entry.keys() + + if not [k for k in entry_keys if get_canSAS_class(entry[k])=='SASdata']: + logger.warning("No sasdata or data key") + logger.warning(f"Known keys: {[k for k in entry_keys]}") + + metadata = parse_metadata(f[root_key]) + + for key in entry_keys: + component = entry[key] + if get_canSAS_class(entry[key])=='SASdata': + datum = recurse_hdf5(component) + data_contents = connected_data(datum, str(filename), metadata) + + if "Qz" in data_contents: + dataset_type = three_dim + elif "Qy" in data_contents: + dataset_type = two_dim + else: + dataset_type = one_dim - entry_key = entry.attrs["sasview_key"] if "sasview_key" in entry.attrs else root_key + entry_key = entry.attrs["sasview_key"] if "sasview_key" in entry.attrs else root_key - loaded_data[entry_key] = SasMeasurement( - name=metadata.title if metadata.title else entry_key, - dataset_type=dataset_type, - data_contents=data_contents, - metadata=metadata, - verbose=False, - ) + loaded_data[entry_key] = SasMeasurement( + name=metadata.title if metadata.title else entry_key, + dataset_type=dataset_type, + data_contents=data_contents, + metadata=metadata, + verbose=False, + ) - return loaded_data + return loaded_data if __name__ == "__main__": - data = load_data(test_file) + data = HDF5Importer().load_data(test_file) for dataset in data.values(): print(dataset.summary()) From 5584da42baa1e17a1ad7971eefce925d550bda20 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:14:49 -0500 Subject: [PATCH 05/25] Convert sesans importer to a child of the Importer class --- sasdata/data_io/importers/import_sesans.py | 32 +++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/sasdata/data_io/importers/import_sesans.py b/sasdata/data_io/importers/import_sesans.py index 460c70bef..f20c49ba0 100644 --- a/sasdata/data_io/importers/import_sesans.py +++ b/sasdata/data_io/importers/import_sesans.py @@ -10,6 +10,7 @@ import numpy as np from sasdata.data import SasMeasurement +from sasdata.data_io.importer import Importer, Source_Type, is_text, read_header from sasdata.data_util.loader_exceptions import FileContentsException from sasdata.dataset_types import sesans from sasdata.metadata import ( @@ -206,7 +207,30 @@ def parse_sesans(lines: list[str]) -> SasMeasurement: ) -def load_data(filename: str | Path) -> SasMeasurement: - with open(filename) as infile: - lines = infile.readlines() - return parse_sesans(lines) +class SesansImporter(Importer): + """Importer for SESANS (.ses) files.""" + + name = "SESANS" + priority = 30 + + @classmethod + def can_import(cls, source: Source_Type) -> bool: + """SESANS files declare their format version in the header of the first line.""" + header = read_header(source) + if not is_text(header): + return False + first_line = header.decode("utf-8", errors="replace").lstrip("\ufeff").splitlines() + return bool(first_line) and re.search(r"FileFormatVersion\s+\S+", first_line[0]) is not None + + def load_data(self, source: Source_Type) -> dict[str, SasMeasurement]: + """Load a SESANS file. The file holds a single measurement, which is keyed by its name.""" + if isinstance(source, (str, Path)): + with open(source) as infile: + lines = infile.readlines() + else: + contents = source.read() + if isinstance(contents, bytes): + contents = contents.decode("utf-8") + lines = contents.splitlines(keepends=True) + measurement = parse_sesans(lines) + return {measurement.name: measurement} From ea4d398256182b9fbfd7766a8cb6c940b90c4a3b Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:15:20 -0500 Subject: [PATCH 06/25] Convert xml importer to a child of the Importer class --- sasdata/data_io/importers/import_xml.py | 140 +++++++++++++++--------- 1 file changed, 86 insertions(+), 54 deletions(-) diff --git a/sasdata/data_io/importers/import_xml.py b/sasdata/data_io/importers/import_xml.py index 6eb557219..c8b02138f 100644 --- a/sasdata/data_io/importers/import_xml.py +++ b/sasdata/data_io/importers/import_xml.py @@ -7,6 +7,7 @@ import sasdata.quantities.unit_parser as unit_parser from sasdata.data import SasMeasurement +from sasdata.data_io.importer import Importer, Source_Type, is_text, read_header from sasdata.dataset_types import one_dim from sasdata.metadata import ( Aperture, @@ -283,62 +284,93 @@ def load_raw(node: etree._Element, version: str) -> MetaNode: return MetaNode(name=etree.QName(node).localname, attrs=attrib, contents=contents) -def load_data(filename: str | Path) -> dict[str, SasMeasurement]: - """Load scattering data from an XML file""" - loaded_data: dict[str, SasMeasurement] = {} - tree = etree.parse(filename) - root = tree.getroot() - - version = get_cansas_version(root) +class XmlImporter(Importer): + """Importer for canSAS 1D XML files (v1.0 and v1.1).""" + + name = "canSAS XML" + priority = 20 + + @classmethod + def can_import(cls, source: Source_Type) -> bool: + """The file is canSAS XML if its root element is a SASroot in one of the canSAS namespaces.""" + header = read_header(source) + if not is_text(header) or not header.lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"<"): + return False + try: + if isinstance(source, (str, Path)): + with open(source, "rb") as f: + return cls._root_is_cansas(f) + try: + return cls._root_is_cansas(source) + finally: + source.seek(0) + except Exception: + return False + + @staticmethod + def _root_is_cansas(handle) -> bool: + """Parse only as far as the root element and check that it is a canSAS SASroot.""" + for _, element in etree.iterparse(handle, events=("start",)): + return get_cansas_version(element) is not None + return False + + def load_data(self, source: Source_Type) -> dict[str, SasMeasurement]: + """Load scattering data from an XML file""" + filename = str(source) if isinstance(source, Path) else source + loaded_data: dict[str, SasMeasurement] = {} + tree = etree.parse(filename) + root = tree.getroot() + + version = get_cansas_version(root) + + if version is None: + logger.error(f"Invalid root: {root.tag!r}") + return loaded_data + + # How many data sets have we loaded? + dataindex = 1 + + for entry in tree.getroot().findall(f"{version}:SASentry", ns): + name = attr_parse(entry, "name") + + title = opt_parse(entry, "Title", version, parse_string) + + if name is None: + if title is None: + name = f"SasData{dataindex:02}" + else: + name = title + + metadata = Metadata( + title=title, + run=all_parse(entry, "Run", version, parse_string), + instrument=opt_parse(entry, "SASinstrument", version, parse_instrument), + process=all_parse(entry, "SASprocess", version, parse_process), + sample=opt_parse(entry, "SASsample", version, parse_sample), + definition=opt_parse(entry, "SASdefinition", version, parse_string), + magnetic=None, # NOTE: Magnetic data isn't part of the XML format. + raw=load_raw(root, version), + ) - if version is None: - logger.error(f"Invalid root: {root.tag!r}") + data = {} + + datacount = 0 + for n in entry.findall(f"{version}:SASdata", ns): + datacount += 1 + data_set = parse_data(n, version, metadata) + data = data_set + break + + loaded_data[name] = SasMeasurement( + name=name, + dataset_type=one_dim, + data_contents=data, + metadata=metadata, + verbose=False, + ) + dataindex += 1 return loaded_data - # How many data sets have we loaded? - dataindex = 1 - - for entry in tree.getroot().findall(f"{version}:SASentry", ns): - name = attr_parse(entry, "name") - - title = opt_parse(entry, "Title", version, parse_string) - - if name is None: - if title is None: - name = f"SasData{dataindex:02}" - else: - name = title - - metadata = Metadata( - title=title, - run=all_parse(entry, "Run", version, parse_string), - instrument=opt_parse(entry, "SASinstrument", version, parse_instrument), - process=all_parse(entry, "SASprocess", version, parse_process), - sample=opt_parse(entry, "SASsample", version, parse_sample), - definition=opt_parse(entry, "SASdefinition", version, parse_string), - magnetic=None, # NOTE: Magnetic data isn't part of the XML format. - raw=load_raw(root, version), - ) - - data = {} - - datacount = 0 - for n in entry.findall(f"{version}:SASdata", ns): - datacount += 1 - data_set = parse_data(n, version, metadata) - data = data_set - break - - loaded_data[name] = SasMeasurement( - name=name, - dataset_type=one_dim, - data_contents=data, - metadata=metadata, - verbose=False, - ) - dataindex += 1 - return loaded_data - if __name__ == "__main__": import sys @@ -350,7 +382,7 @@ def load_data(filename: str | Path) -> dict[str, SasMeasurement]: for f in files: print(f) - data = load_data(f) + data = XmlImporter().load_data(f) for dataset in data.values(): print(dataset.summary()) From 5a307ea83a2c1c701c09453144ddf3b922de824e Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:18:53 -0500 Subject: [PATCH 07/25] Register all importers in the base importers package to load on import --- sasdata/data_io/importers/__init__.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sasdata/data_io/importers/__init__.py b/sasdata/data_io/importers/__init__.py index 2134dd59e..ab359039f 100644 --- a/sasdata/data_io/importers/__init__.py +++ b/sasdata/data_io/importers/__init__.py @@ -1 +1,15 @@ -__all__ = ['import_ascii', 'import_hdf5', 'import_sesans', 'import_xml'] +""" +Importers for each of the file formats supported by sasdata. + +Each module defines a child of :class:`sasdata.data_io.importer.Importer`. Importing this package registers all of +them, so :meth:`Importer.find_importer` can select the correct one from the contents of a file. +""" + +from sasdata.data_io.importers.import_ascii import AsciiImporter +from sasdata.data_io.importers.import_hdf5 import HDF5Importer +from sasdata.data_io.importers.import_sesans import SesansImporter +from sasdata.data_io.importers.import_xml import XmlImporter + +__all__ = [ + 'AsciiImporter', 'HDF5Importer', 'SesansImporter', 'XmlImporter', +] From 00a87365a7636b31a90f28453d6d375d2842ea36 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:21:39 -0500 Subject: [PATCH 08/25] Simplify the local_load for test files in the base test directory --- test/__init__.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index 4c83a99cf..bacf715ed 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -2,21 +2,16 @@ import pathlib from sasdata.data import SasData -from sasdata.data_io.importers.import_ascii import load_data_default_params -from sasdata.data_io.importers.import_hdf5 import load_data as hdf_load_data -from sasdata.data_io.importers.import_xml import load_data as xml_load_data +from sasdata.data_io.importer import Importer from test.test_files import data, json, mumag, reference, sesans_data -def local_load(path: str) -> SasData: - """Get local file path""" +def local_load(path: str) -> list[SasData]: + """Load a local file, letting the Importer infer the file type from its contents""" base = os.path.join(os.path.dirname(__file__), path) - if os.path.exists(f"{base}.h5"): - return hdf_load_data(f"{base}.h5").values() - if os.path.exists(f"{base}.xml"): - return xml_load_data(f"{base}.xml").values() - if os.path.exists(f"{base}.txt"): - return load_data_default_params(f"{base}.txt") + for extension in ("h5", "xml", "txt"): + if os.path.exists(f"{base}.{extension}"): + return list(Importer.load(f"{base}.{extension}").values()) def local_reference_finder(path: str): From 97e139a1ffd25af3ebd823c89addd822a51aea88 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:22:38 -0500 Subject: [PATCH 09/25] Update averaging unit tests to use new Importer classes, as needed --- test/sasmanipulations/utest_averaging.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/sasmanipulations/utest_averaging.py b/test/sasmanipulations/utest_averaging.py index e834c9d80..2ab8a5040 100644 --- a/test/sasmanipulations/utest_averaging.py +++ b/test/sasmanipulations/utest_averaging.py @@ -3,8 +3,8 @@ import numpy as np from sasdata.data import SasMeasurement, sasdata_reader2D_converter -from sasdata.data_io.importers.import_ascii import load_data_default_params as ascii_load_data -from sasdata.data_io.importers.import_hdf5 import load_data as hdf_load_data +from sasdata.data_io.importers.import_ascii import AsciiImporter +from sasdata.data_io.importers.import_hdf5 import HDF5Importer from sasdata.data_util.manipulations import ( Boxavg, Boxsum, @@ -131,7 +131,7 @@ class DataInfoTests(unittest.TestCase): def setUp(self): filepath = find('MAR07232_rest.h5') - data_dict = hdf_load_data(filepath) + data_dict = HDF5Importer().load_data(filepath) self.data = data_dict['sasentry01'] def test_ring(self): @@ -151,7 +151,7 @@ def test_ring(self): o = r(self.data) filepath = find('ring_testdata.txt') - answer_list = ascii_load_data(filepath) + answer_list = list(AsciiImporter().load_data(filepath).values()) answer = answer_list[0] self.assertEqual(len(answer_list), 1) @@ -173,7 +173,7 @@ def test_circularavg(self): o = r(self.data) filepath = find('avg_testdata.txt') - answer = ascii_load_data(filepath)[0] + answer = list(AsciiImporter().load_data(filepath).values())[0] for i in range(r.nbins_phi): self.assertAlmostEqual(o.abscissae.axes[0].value[i], answer.abscissae.axes[0].value[i], delta=1e-4) self.assertAlmostEqual(o.ordinate.value[i], answer.ordinate.value[i], delta=1e-4) @@ -208,7 +208,7 @@ def test_slabX(self): o = r(self.data) filepath = find('slabx_testdata.txt') - answer = ascii_load_data(filepath)[0] + answer = list(AsciiImporter().load_data(filepath).values())[0] for i in range(len(o.abscissae.axes[0].value)): self.assertAlmostEqual(o.abscissae.axes[0].value[i], answer.abscissae.axes[0].value[i], 4) self.assertAlmostEqual(o.ordinate.value[i], answer.ordinate.value[i], 4) @@ -226,7 +226,7 @@ def test_slabY(self): o = r(self.data) filepath = find('slaby_testdata.txt') - answer = ascii_load_data(filepath)[0] + answer = list(AsciiImporter().load_data(filepath).values())[0] for i in range(len(o.abscissae.axes[0].value)): self.assertAlmostEqual(o.abscissae.axes[0].value[i], answer.abscissae.axes[0].value[i], 4) self.assertAlmostEqual(o.ordinate.value[i], answer.ordinate.value[i], 4) @@ -252,7 +252,7 @@ def test_sectorphi_full(self): o = r(self.data) filepath = find('ring_testdata.txt') - answer = ascii_load_data(filepath)[0] + answer = list(AsciiImporter().load_data(filepath).values())[0] for i in range(len(o.abscissae.axes[0].value)-1): self.assertAlmostEqual(o.abscissae.axes[0].value[i], answer.abscissae.axes[0].value[i+1], 4) self.assertAlmostEqual(o.ordinate.value[i], answer.ordinate.value[i+1], 4) @@ -269,7 +269,7 @@ def test_sectorphi_quarter(self): o = r(self.data) filepath = find('sectorphi_testdata.txt') - answer = ascii_load_data(filepath)[0] + answer = list(AsciiImporter().load_data(filepath).values())[0] for i in range(len(o.abscissae.axes[0].value)): self.assertAlmostEqual(o.abscissae.axes[0].value[i], answer.abscissae.axes[0].value[i], 4) self.assertAlmostEqual(o.ordinate.value[i], answer.ordinate.value[i], 4) @@ -286,7 +286,7 @@ def test_sectorq_full(self): o = r(self.data) filepath = find('sectorq_testdata.txt') - answer = ascii_load_data(filepath)[0] + answer = list(AsciiImporter().load_data(filepath).values())[0] for i in range(len(o.abscissae.axes[0].value)): self.assertAlmostEqual(o.abscissae.axes[0].value[i], answer.abscissae.axes[0].value[i], 4) self.assertAlmostEqual(o.ordinate.value[i], answer.ordinate.value[i], 4) From 67081ae78a80c3d7ae155f516b3b836d13291a10 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:24:04 -0500 Subject: [PATCH 10/25] Remove undefined pytest mark to fix worning --- test/data_io/utest_data_names.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/data_io/utest_data_names.py b/test/data_io/utest_data_names.py index b83c28117..61cad0ce0 100644 --- a/test/data_io/utest_data_names.py +++ b/test/data_io/utest_data_names.py @@ -16,7 +16,6 @@ ] -@pytest.mark.names @pytest.mark.parametrize("x", test_file_names) def test_quantity_name(x): (f, expected) = x From adc5fb902c5ae3cf5508522a4834f61d62dc8ec4 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:25:17 -0500 Subject: [PATCH 11/25] Rename TestFileReader to MockFileReader so pytest does not mistake it for a unit test suite --- test/data_io/utest_generic_file_import_class.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/data_io/utest_generic_file_import_class.py b/test/data_io/utest_generic_file_import_class.py index 4bb78903b..e1eb6bacc 100644 --- a/test/data_io/utest_generic_file_import_class.py +++ b/test/data_io/utest_generic_file_import_class.py @@ -24,7 +24,7 @@ def find(filename): class GenericFileReaderTests(unittest.TestCase): def setUp(self): - self.reader = TestFileReader() + self.reader = MockFileReader() self.bad_file = find("ACB123.txt") self.good_file = find("123ABC.txt") self.generic_reader = Loader() @@ -92,7 +92,7 @@ def tearDown(self): if os.path.isfile(self.good_file): os.remove(self.good_file) -class TestFileReader(FileReader): +class MockFileReader(FileReader): def get_file_contents(self): """ Reader specific class to access the contents of the file From 5dd87603c72c894436d1654b05bbf9b311813af4 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:26:48 -0500 Subject: [PATCH 12/25] Update generic file import tests to use new Importer class and children --- .../utest_generic_file_import_class.py | 58 +++++++++++++++++-- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/test/data_io/utest_generic_file_import_class.py b/test/data_io/utest_generic_file_import_class.py index e1eb6bacc..6fd07bab1 100644 --- a/test/data_io/utest_generic_file_import_class.py +++ b/test/data_io/utest_generic_file_import_class.py @@ -7,8 +7,10 @@ import unittest import numpy as np +import pytest from sasdata.data_io.importer import Importer +from sasdata.data_io.importers import AsciiImporter, HDF5Importer, SesansImporter, XmlImporter from sasdata.dataloader.data_info import Data1D, DataInfo, plottable_1D from sasdata.dataloader.filereader import FileReader from sasdata.dataloader.loader import Loader @@ -110,14 +112,58 @@ class TestImporterClass: def test_basic_import(self): file_list = [ find("TestExtensions.xml"), - find("FEB18012.ASC"), # Silently Fails - find("Anton-Paar.pdh"), # Silently Fails - find("ascii_test_1.txt"), # Silently Fails - find("MAR07232_rest.h5"), # Silently Fails + find("FEB18012.ASC"), + find("Anton-Paar.pdh"), + find("ascii_test_1.txt"), + find("MAR07232_rest.h5"), find("nans_in_2d_data.DAT") ] importer = Importer() imported_data, errors = importer.import_data(file_list) - assert len(imported_data) == 1 + assert len(imported_data) == 6 assert imported_data[0].name == "TK49 c10_SANS" - assert len(errors) == 2 + assert imported_data[4].name == "MAR07232_rest_out.dat" + assert errors == [] + + def test_missing_file(self): + missing = find("ACB123.txt") + imported_data, errors = Importer().import_data([missing, find("TestExtensions.xml")]) + assert len(imported_data) == 1 + assert len(errors) == 1 + assert "File does not exist" in errors[0] + + @pytest.mark.parametrize("filename, expected_importer", [ + ("TestExtensions.xml", XmlImporter), + ("FEB18012.ASC", AsciiImporter), + ("Anton-Paar.pdh", AsciiImporter), + ("ascii_test_1.txt", AsciiImporter), + ("MAR07232_rest.h5", HDF5Importer), + ("nans_in_2d_data.DAT", AsciiImporter), + ("ACB123.txt", None), + ]) + def test_find_importer(self, filename, expected_importer): + assert Importer.find_importer(find(filename)) is expected_importer + + @pytest.mark.parametrize("filename", [ + "TestExtensions", "TestExtensions.asc", "TestExtensions.nxs", "TestExtensions.notxml", "TestExtensions.xml", + ]) + def test_file_type_inferred_from_contents(self, filename): + """The file extension should not matter: the importer is chosen from the file contents""" + assert Importer.find_importer(find(filename)) is XmlImporter + data = Importer.load(find(filename)) + assert list(data.keys()) == ["TK49 c10_SANS"] + measurement = data["TK49 c10_SANS"] + assert len(measurement.ordinate.value) == 138 + assert measurement.metadata.sample.sample_id == "TK49 c10_SANS" + + def test_registry(self): + importers = Importer.importers() + for importer in (HDF5Importer, XmlImporter, SesansImporter, AsciiImporter): + assert importer in importers + assert issubclass(importer, Importer) + # The generic ASCII importer must be the last resort + assert importers[-1] is AsciiImporter + + def test_unknown_file_type(self): + with pytest.raises(ValueError): + Importer().load_data(find("ACB123.txt")) From 0a9ce7b490a702eb4855ed1e804092bdf23a973f Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:27:44 -0500 Subject: [PATCH 13/25] Update trend generation tests to use new Importer class and children --- test/utest_trend.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/utest_trend.py b/test/utest_trend.py index 8d38cdb19..a685f048e 100644 --- a/test/utest_trend.py +++ b/test/utest_trend.py @@ -3,11 +3,10 @@ import numpy as np import pytest -import sasdata.data_io.importers.import_ascii as ascii_reader from sasdata.ascii_reader_metadata import AsciiMetadataCategory -from sasdata.data_io.importers.import_ascii import AsciiReaderParams -from sasdata.data_io.importers.import_hdf5 import load_data as hdf_load_data -from sasdata.data_io.importers.import_xml import load_data as xml_load_data +from sasdata.data_io.importers.import_ascii import AsciiImporter, AsciiReaderParams +from sasdata.data_io.importers.import_hdf5 import HDF5Importer +from sasdata.data_io.importers.import_xml import XmlImporter from sasdata.quantities.units import per_angstrom, per_nanometer from sasdata.trend import Trend from test import local_data_finder, local_mumag_finder @@ -40,7 +39,7 @@ def test_trend_build_from_xml(): and currently a single Metadata (no tree of MetaNodes), but this should be corrected in the future in the XML reader, I guess! """ - data = xml_load_data(xml_file) # dict of SasData objects + data = XmlImporter().load_data(xml_file) # dict of SasData objects trend = Trend(data=[data["SasData01"]], trend_axes={"entry": ["SASentry"]}) assert not trend.is_manual_axis("entry") assert len(trend.get_trend_values("entry")) == 1 @@ -52,7 +51,7 @@ def test_trend_build_from_hdf5_with_multiple_axes(): The loader returns a dict of SasData objects (here two, named 'sasentry01' and 'sasentry02') Need to think how to compare the Quantity objects """ - data = hdf_load_data(hdf_file) # dict of SasData objects + data = HDF5Importer().load_data(hdf_file) # dict of SasData objects trend = Trend( data=list(data.values()), trend_axes={ @@ -90,7 +89,7 @@ def test_trend_build_from_ascii_with_manual_axis(directory_name: str): "demagnetizing_field": 3, } ) - data = ascii_reader.load_data(params) + data = list(AsciiImporter(params=params).load_data().values()) trend = Trend( data=data, trend_axes={ @@ -125,7 +124,7 @@ def test_trend_build_interpolate(directory_name: str): "demagnetizing_field": 3, } ) - data = ascii_reader.load_data(params) + data = list(AsciiImporter(params=params).load_data().values()) trend = Trend(data=data, trend_axes={"field": ["magnetic", "applied_magnetic_field"]}) # Initially, the q axes in this data don't exactly match to_interpolate_on = "Q" @@ -148,6 +147,6 @@ def test_trend_q_axis_match(): "counting_index": 0, } ) - data = ascii_reader.load_data(params) + data = list(AsciiImporter(params=params).load_data().values()) trend = Trend(data=data, trend_axes={"index": ["magnetic", "counting_index"]}) assert trend.all_axis_match("Q") From 30539863cdda643b6639a2bc5127db5d923893bf Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:29:11 -0500 Subject: [PATCH 14/25] Update ascii importer tests to use new AsciiImporter class --- test/data_io/utest_import_ascii.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/data_io/utest_import_ascii.py b/test/data_io/utest_import_ascii.py index 28d0f3496..12fe77edb 100644 --- a/test/data_io/utest_import_ascii.py +++ b/test/data_io/utest_import_ascii.py @@ -4,10 +4,9 @@ from sasdata.ascii_reader_metadata import AsciiMetadataCategory, AsciiReaderMetadata from sasdata.data_io.importers.import_ascii import ( + AsciiImporter, AsciiReaderParams, guess_params_from_filename, - load_data, - load_data_default_params, ) from sasdata.dataset_types import one_dim from sasdata.guess import guess_columns @@ -42,7 +41,7 @@ def test_ascii_1(): ("", None), ("", None), ] - loaded_data = load_data(params)[0] + loaded_data = list(AsciiImporter(params=params).load_data().values())[0] assert len(loaded_data._data_contents) > 0 # Check the first, and last rows to see if they are correct. for name, datum in loaded_data._data_contents.items(): @@ -60,7 +59,7 @@ def test_ascii_1(): def test_ascii_2(): filename = find("test_3_columns.txt", "sasdataloader") - loaded_data = load_data_default_params(filename)[0] + loaded_data = AsciiImporter().load_data(filename)["test_3_columns.txt"] assert len(loaded_data._data_contents) > 0 for name, datum in loaded_data._data_contents.items(): match name: @@ -81,7 +80,7 @@ def test_ascii_2(): def test_ascii_2d(): filename = find("detector_rectangular.DAT", "sasdataloader") # Make sure that the dataset type is guessed as 2D data. - loaded_data = load_data_default_params(filename)[0] + loaded_data = list(AsciiImporter().load_data(filename).values())[0] assert len(loaded_data._data_contents) > 0 for name, datum in loaded_data._data_contents.items(): @@ -131,8 +130,9 @@ def test_mumag_metadata(): separator_dict={"Comma": True}, metadata=metadata, ) - data = load_data(params) - for datum in data: + data = AsciiImporter(params=params).load_data() + assert len(data) == len(filenames) + for datum in data.values(): match datum.name: case "1_33_1640_22.874115.csv": assert datum.metadata.raw.filter("counting_index") == ["1"] From 5b4f3943ad2b2b6dbcbdde4c46c606eb64abe534 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:30:50 -0500 Subject: [PATCH 15/25] Update sesans importer tests to use new Importer class and children --- test/data_io/utest_import_sesans.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/data_io/utest_import_sesans.py b/test/data_io/utest_import_sesans.py index 59a3d3dce..47396484c 100644 --- a/test/data_io/utest_import_sesans.py +++ b/test/data_io/utest_import_sesans.py @@ -6,7 +6,8 @@ import numpy as np import pytest -from sasdata.data_io.importers.import_sesans import load_data +from sasdata.data_io.importer import Importer +from sasdata.data_io.importers.import_sesans import SesansImporter from sasdata.model_requirements import ( ComposeRequirements, ModellingRequirements, @@ -26,15 +27,28 @@ @pytest.mark.sesans @pytest.mark.parametrize("f", test_file_names) def test_load_file(f): - data = load_data(local_sesans_finder(f"{f}.ses")) + loaded = SesansImporter().load_data(local_sesans_finder(f"{f}.ses")) + assert len(loaded) == 1 + data = loaded["Sesans"] with open(local_reference_finder(f"{f}.txt")) as infile: expected = "".join(infile.readlines()) assert data.summary() == expected +@pytest.mark.sesans +@pytest.mark.parametrize("f", test_file_names) +def test_sesans_importer_inferred(f): + """The generic importer recognises SESANS files from their contents""" + filename = local_sesans_finder(f"{f}.ses") + assert Importer.find_importer(filename) is SesansImporter + assert SesansImporter.can_import(filename) + inferred = Importer.load(filename) + direct = SesansImporter().load_data(filename) + assert [d.summary() for d in inferred.values()] == [d.summary() for d in direct.values()] + @pytest.mark.sesans def test_sesans_modelling(): - data = load_data(local_sesans_finder("sphere2micron.ses")) + data = SesansImporter().load_data(local_sesans_finder("sphere2micron.ses"))["Sesans"] req = guess_requirements(data) assert type(req) is SesansModel From 557757fed7c4f5c9f295ffd9fce07d91f61ba3db Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:32:35 -0500 Subject: [PATCH 16/25] Remove unrecognized pytest mark to eliminate unit test warnings --- test/data_io/utest_sasdata_import.py | 61 ++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/test/data_io/utest_sasdata_import.py b/test/data_io/utest_sasdata_import.py index 13a20c033..aadc64c05 100644 --- a/test/data_io/utest_sasdata_import.py +++ b/test/data_io/utest_sasdata_import.py @@ -12,16 +12,16 @@ import sasdata.quantities.units as units from sasdata.data import SasData, SasDataEncoder, SasMeasurement +from sasdata.data_io.importer import Importer from sasdata.data_io.importers.import_ascii import ( + AsciiImporter, AsciiMetadataCategory, AsciiReaderMetadata, AsciiReaderParams, - load_data_default_params, ) -from sasdata.data_io.importers.import_ascii import load_data as ascii_load_data -from sasdata.data_io.importers.import_hdf5 import load_data as hdf_load_data -from sasdata.data_io.importers.import_sesans import load_data as sesans_load_data -from sasdata.data_io.importers.import_xml import load_data as xml_load_data +from sasdata.data_io.importers.import_hdf5 import HDF5Importer +from sasdata.data_io.importers.import_sesans import SesansImporter +from sasdata.data_io.importers.import_xml import XmlImporter from sasdata.dataset_types import one_dim from sasdata.guess import guess_columns from sasdata.quantities.quantity import Quantity @@ -31,12 +31,12 @@ @pytest.mark.sasdata def test_filter_data(): - data = xml_load_data(local_data_finder("cansas1d_notitle.xml")) + data = XmlImporter().load_data(local_data_finder("cansas1d_notitle.xml")) for k, v in data.items(): assert v.metadata.raw.filter("transmission") == ["0.327"] assert v.metadata.raw.filter("wavelength")[0] == Quantity(6.0, units.angstroms) assert v.metadata.raw.filter("SDD")[0] == Quantity(4.15, units.meters) - data = hdf_load_data(local_data_finder("nxcansas_1Dand2D_multisasentry.h5")) + data = HDF5Importer().load_data(local_data_finder("nxcansas_1Dand2D_multisasentry.h5")) for k, v in data.items(): assert v.metadata.raw.filter("radiation") == ["Spallation Neutron Source"] assert v.metadata.raw.filter("SDD") == [ @@ -336,25 +336,25 @@ def is_uncertainty(column: str) -> bool: def test_load_file(test_case: BaseTestCase): match test_case: case BulkAsciiTestCase(): - loaded_data = ascii_load_data(test_case.reader_params) + loaded_data = list(AsciiImporter(params=test_case.reader_params).load_data().values()) case AsciiTestCase(): if isinstance(test_case.reader_params, str): - loaded_data = load_data_default_params(test_case.reader_params)[0] + loaded_data = list(AsciiImporter().load_data(test_case.reader_params).values())[0] elif isinstance(test_case.reader_params, AsciiReaderParams): - loaded_data = ascii_load_data(test_case.reader_params)[0] + loaded_data = list(AsciiImporter(params=test_case.reader_params).load_data().values())[0] else: raise TypeError("Invalid type for reader_params.") case Hdf5TestCase(): - combined_data = hdf_load_data(test_case.filename) + combined_data = HDF5Importer().load_data(test_case.filename) loaded_data = combined_data[test_case.entry] - # TODO: Support SESANS case XmlTestCase(): # Not bulk, so just assume we get one dataset. - combined_data = xml_load_data(test_case.filename) + combined_data = XmlImporter().load_data(test_case.filename) loaded_data = combined_data[test_case.entry] case SesansTestCase(): - loaded_data = sesans_load_data(test_case.filename) - combined_data = {"only": loaded_data} + combined_data = SesansImporter().load_data(test_case.filename) + assert len(combined_data) == 1 + loaded_data = next(iter(combined_data.values())) case _: raise ValueError("Invalid loader") if isinstance(test_case, BulkAsciiTestCase): @@ -414,7 +414,8 @@ def test_load_file(test_case: BaseTestCase): SasMeasurement.save_h5(combined_data, bio) bio.seek(0) - result = hdf_load_data(bio) + assert HDF5Importer.can_import(bio) + result = HDF5Importer().load_data(bio) bio.close() for name, entry in result.items(): @@ -428,3 +429,31 @@ def test_load_file(test_case: BaseTestCase): assert np.all(combined_data[name].ordinate.value == entry.ordinate.value) assert combined_data[name].abscissae.units == entry.abscissae.units assert np.all(combined_data[name].abscissae.value == entry.abscissae.value) + + +expected_importers: dict[type[BaseTestCase], type[Importer]] = { + XmlTestCase: XmlImporter, + Hdf5TestCase: HDF5Importer, + SesansTestCase: SesansImporter, +} +inference_cases = [ + (case.filename, expected_importers[type(case)]) + for case in test_cases + if type(case) in expected_importers +] + [ + (local_data_finder("test_3_columns.txt"), AsciiImporter), + (local_data_finder("ascii_test_1.txt"), AsciiImporter), +] + + +@pytest.mark.dataload +@pytest.mark.parametrize("filename, expected_importer", inference_cases) +def test_infer_importer(filename: str, expected_importer: type[Importer]): + """The generic Importer selects the right child importer from the file contents""" + assert Importer.find_importer(filename) is expected_importer + assert expected_importer.can_import(filename) + # Data loaded through the generic importer should be identical to using the specific importer directly. + inferred = Importer.load(filename) + direct = expected_importer().load_data(filename) + assert list(inferred.keys()) == list(direct.keys()) + assert [d.summary() for d in inferred.values()] == [d.summary() for d in direct.values()] From 6803ee1321f78fd86e9c0486021525f46f7eae2f Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 4 Nov 2026 10:34:36 -0500 Subject: [PATCH 17/25] Remove unrecognized pytest mark to eliminate test warnings --- test/data_io/utest_import_sesans.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/data_io/utest_import_sesans.py b/test/data_io/utest_import_sesans.py index 47396484c..62d4ddae2 100644 --- a/test/data_io/utest_import_sesans.py +++ b/test/data_io/utest_import_sesans.py @@ -24,7 +24,6 @@ test_file_names = ["sphere2micron", "sphere_isis"] -@pytest.mark.sesans @pytest.mark.parametrize("f", test_file_names) def test_load_file(f): loaded = SesansImporter().load_data(local_sesans_finder(f"{f}.ses")) @@ -35,7 +34,6 @@ def test_load_file(f): expected = "".join(infile.readlines()) assert data.summary() == expected -@pytest.mark.sesans @pytest.mark.parametrize("f", test_file_names) def test_sesans_importer_inferred(f): """The generic importer recognises SESANS files from their contents""" @@ -46,7 +44,6 @@ def test_sesans_importer_inferred(f): direct = SesansImporter().load_data(filename) assert [d.summary() for d in inferred.values()] == [d.summary() for d in direct.values()] -@pytest.mark.sesans def test_sesans_modelling(): data = SesansImporter().load_data(local_sesans_finder("sphere2micron.ses"))["Sesans"] req = guess_requirements(data) @@ -63,11 +60,9 @@ def form_volume(x): xi_squared = smear(req, data._data_contents["SpinEchoLength"].value, full_data=data, y = data._data_contents["Depolarisation"].in_units_of(unit_parser.parse("A-2 cm-1")) / f2, radius=radius) assert 1.0 < xi_squared < 1.5 -@pytest.mark.sesans def test_pinhole_zero(): assert pinhole_smear(0) == 0 -@pytest.mark.sesans def test_pinhole_smear(): smearing = [3**x for x in range(-1, 6)] smears = [pinhole_smear(x) for x in smearing] @@ -78,11 +73,9 @@ def test_pinhole_smear(): old = smear assert pinhole_smear(3**6) > 1.2 -@pytest.mark.sesans def test_slit_zero(): assert slit_smear(0, 0) == 0 -@pytest.mark.sesans def test_slit_smear(): smears = [(length, width, slit_smear(3**length, 3**width)) for length in range(-1, 6) for width in range(-1, 6) if length + width == 6] for length, width, smear in smears: @@ -123,7 +116,6 @@ def sas_3j1x_x(x): -@pytest.mark.sesans def test_model_algebra(): ses = SesansModel() pin = PinholeModel(np.linspace(1e-3, 1, 1000)) From 846d95980c941a754adf961f613357cefc2a5b60 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 08:57:27 -0500 Subject: [PATCH 18/25] Use importer.Path_Type for type checks in child classes --- sasdata/data_io/importers/import_ascii.py | 7 +++---- sasdata/data_io/importers/import_xml.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index b467bc191..f811e3fe8 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field, replace from enum import Enum from os import path -from pathlib import Path import numpy as np @@ -13,7 +12,7 @@ pairings, ) from sasdata.data import SasMeasurement -from sasdata.data_io.importer import Importer, Source_Type, is_text, measurements_by_name, read_header +from sasdata.data_io.importer import Importer, Path_Type, Source_Type, is_text, measurements_by_name, read_header from sasdata.dataset_types import DatasetType, one_dim, unit_kinds from sasdata.default_units import get_default_unit from sasdata.guess import ( @@ -239,7 +238,7 @@ def __init__(self, config: dict | None = None, params: AsciiReaderParams | None @classmethod def can_import(cls, source: Source_Type) -> bool: """Any text file on disk could be a column-based ASCII file.""" - if not isinstance(source, (str, Path)): + if not isinstance(source, Path_Type): return False return is_text(read_header(source)) @@ -255,7 +254,7 @@ def load_data(self, source: Source_Type | None = None) -> dict[str, SasMeasureme if self.params is None: raise ValueError("A file to load, or AsciiReaderParams listing the files to load, must be given.") params = self.params - elif not isinstance(source, (str, Path)): + elif not isinstance(source, Path_Type): raise TypeError("The ASCII importer can only load files from a path.") elif self.params is not None: params = replace(self.params, filenames=[str(source)]) diff --git a/sasdata/data_io/importers/import_xml.py b/sasdata/data_io/importers/import_xml.py index c8b02138f..9a4738253 100644 --- a/sasdata/data_io/importers/import_xml.py +++ b/sasdata/data_io/importers/import_xml.py @@ -7,7 +7,7 @@ import sasdata.quantities.unit_parser as unit_parser from sasdata.data import SasMeasurement -from sasdata.data_io.importer import Importer, Source_Type, is_text, read_header +from sasdata.data_io.importer import Importer, Path_Type, Source_Type, is_text, read_header from sasdata.dataset_types import one_dim from sasdata.metadata import ( Aperture, @@ -297,7 +297,7 @@ def can_import(cls, source: Source_Type) -> bool: if not is_text(header) or not header.lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"<"): return False try: - if isinstance(source, (str, Path)): + if isinstance(source, Path_Type): with open(source, "rb") as f: return cls._root_is_cansas(f) try: From cf9b0324d729777519621fd04d9b2d23bb6e57ff Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 08:58:10 -0500 Subject: [PATCH 19/25] Use default sniff size for hdf5 reader instead of magic number --- sasdata/data_io/importers/import_hdf5.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sasdata/data_io/importers/import_hdf5.py b/sasdata/data_io/importers/import_hdf5.py index 6fb1ccf46..a641c8470 100644 --- a/sasdata/data_io/importers/import_hdf5.py +++ b/sasdata/data_io/importers/import_hdf5.py @@ -10,7 +10,7 @@ from sasdata.data import SasMeasurement from sasdata.data_backing import Dataset as SASDataDataset from sasdata.data_backing import Group as SASDataGroup -from sasdata.data_io.importer import Importer, Source_Type, read_header +from sasdata.data_io.importer import SNIFF_SIZE, Importer, Source_Type, read_header from sasdata.dataset_types import one_dim, three_dim, two_dim from sasdata.metadata import ( Aperture, @@ -30,11 +30,11 @@ from sasdata.quantities.quantity import NamedQuantity, Quantity from sasdata.quantities.unit_parser import parse -# test_file = "../example_data/1d_data/33837rear_1D_1.75_16.5_NXcanSAS_v3.h5" -# test_file = "../example_data/1d_data/33837rear_1D_1.75_16.5_NXcanSAS.h5" +# test_file = "../../../sasdata/example_data/1d_data/33837rear_1D_1.75_16.5_NXcanSAS_v3.h5" +# test_file = "../../../sasdata/example_data/1d_data/33837rear_1D_1.75_16.5_NXcanSAS.h5" test_file = "../../../sasdata/example_data/2d_data/BAM_2D.h5" -# test_file = "../example_data/2d_data/14250_2D_NoDetInfo_NXcanSAS_v3.h5" -# test_file = "../example_data/2d_data/33837rear_2D_1.75_16.5_NXcanSAS_v3.h5" +# test_file = "../../../sasdata/example_data/2d_data/14250_2D_NoDetInfo_NXcanSAS_v3.h5" +# test_file = "../../../sasdata/example_data/2d_data/33837rear_2D_1.75_16.5_NXcanSAS_v3.h5" #test_file = "../../../test/test_files/data/nxcansas_1Dand2D_multisasdata.h5" logger = logging.getLogger(__name__) @@ -426,7 +426,7 @@ def can_import(cls, source: Source_Type) -> bool: return h5py.is_hdf5(source) except Exception: return False - header = read_header(source, size=4096) + header = read_header(source, size=SNIFF_SIZE) offset = 0 while offset + len(HDF5_SIGNATURE) <= len(header): if header[offset:offset + len(HDF5_SIGNATURE)] == HDF5_SIGNATURE: From 10aad2a4724fc06d642b509e46896a3b4da480d9 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 08:59:04 -0500 Subject: [PATCH 20/25] Check for explicit errors when checking if a file is importable instead of any error --- sasdata/data_io/importers/import_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sasdata/data_io/importers/import_xml.py b/sasdata/data_io/importers/import_xml.py index 9a4738253..46f52f1dc 100644 --- a/sasdata/data_io/importers/import_xml.py +++ b/sasdata/data_io/importers/import_xml.py @@ -304,7 +304,7 @@ def can_import(cls, source: Source_Type) -> bool: return cls._root_is_cansas(source) finally: source.seek(0) - except Exception: + except (FileNotFoundError, ValueError): return False @staticmethod From 1930b72b44f68dbf580073eb2073770e9e3980fc Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 08:59:57 -0500 Subject: [PATCH 21/25] Check for cases where more configs are passed to the Importer than files and break out of the loop in such a case --- sasdata/data_io/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index c732bb5c8..bcf6b6550 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -121,7 +121,8 @@ def import_data(self, files: list[Path_Type], configs: list[dict[str, Value_Type configs = [] fill_value = configs[-1] if len(configs) > 0 else None for file, config in zip_longest(files, configs, fillvalue=fill_value): - if file is None: + if file == fill_value: + # If this is reached, the most likely scenario is len(configs) > len(files) in which case, break out of the loop break measurements, file_errors = self._import_file(Path(file), config) output.extend(measurements) From 5e9ec80978fd29b053a4edf8e881320ab203405c Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 09:00:58 -0500 Subject: [PATCH 22/25] Change import test local_load to be file extension agnostic, like it should be --- test/__init__.py | 7 +++---- test/data_io/utest_data_names.py | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index bacf715ed..3bec9cac9 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -8,10 +8,9 @@ def local_load(path: str) -> list[SasData]: """Load a local file, letting the Importer infer the file type from its contents""" - base = os.path.join(os.path.dirname(__file__), path) - for extension in ("h5", "xml", "txt"): - if os.path.exists(f"{base}.{extension}"): - return list(Importer.load(f"{base}.{extension}").values()) + f_path = pathlib.Path(path) + if f_path.exists() and f_path.is_file(): + return list(Importer.load(f_path).values()) def local_reference_finder(path: str): diff --git a/test/data_io/utest_data_names.py b/test/data_io/utest_data_names.py index 61cad0ce0..c8be70f81 100644 --- a/test/data_io/utest_data_names.py +++ b/test/data_io/utest_data_names.py @@ -8,11 +8,11 @@ from test import local_data_finder, local_load test_file_names = [ - ("ascii_test_1", "::Q:3KrS58TPgclJ1rgyr0VQp3"), - ("ISIS_1_1", "TK49 c10_SANS:79680:Q:4TghWEoJi6xxhyeDXhS751"), - ("cansas1d", "Test title:1234:Q:440tNBqdx9jvci6CgjmrmD"), - ("MAR07232_rest", "MAR07232_rest_out.dat:2:/sasentry01/sasdata01/Qx:37t0tPj1o8oQcQEB3DVUlw"), - ("simpleexamplefile", "::/sasentry01/sasdata01/Q:uoHMeB8mukElC1uLCy7Sd"), + ("ascii_test_1.txt", "::Q:3KrS58TPgclJ1rgyr0VQp3"), + ("ISIS_1_1.xml", "TK49 c10_SANS:79680:Q:4TghWEoJi6xxhyeDXhS751"), + ("cansas1d.xml", "Test title:1234:Q:440tNBqdx9jvci6CgjmrmD"), + ("MAR07232_rest.h5", "MAR07232_rest_out.dat:2:/sasentry01/sasdata01/Qx:37t0tPj1o8oQcQEB3DVUlw"), + ("simpleexamplefile.h5", "::/sasentry01/sasdata01/Q:uoHMeB8mukElC1uLCy7Sd"), ] From 238d44568a083be0709d4e1a1717cca63e1b5cf9 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 09:01:27 -0500 Subject: [PATCH 23/25] Be more explicit with the importer priorities tests --- test/data_io/utest_generic_file_import_class.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/data_io/utest_generic_file_import_class.py b/test/data_io/utest_generic_file_import_class.py index 6fd07bab1..430a4a06a 100644 --- a/test/data_io/utest_generic_file_import_class.py +++ b/test/data_io/utest_generic_file_import_class.py @@ -161,7 +161,9 @@ def test_registry(self): for importer in (HDF5Importer, XmlImporter, SesansImporter, AsciiImporter): assert importer in importers assert issubclass(importer, Importer) - # The generic ASCII importer must be the last resort + # Check priorities are in proper order and the Ascii Importer is always a last resort + priorities = [importer.priority for importer in importers] + assert all(x < y for x, y in zip(priorities, priorities[1:])) assert importers[-1] is AsciiImporter def test_unknown_file_type(self): From 1a269dfa59fc6693ca17bb89956534982ee8ea58 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 8 Nov 2026 09:12:44 -0500 Subject: [PATCH 24/25] Document why exceptions are swallowed --- sasdata/data_io/importer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index bcf6b6550..e16f8122b 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -157,7 +157,7 @@ def read_header(source: Source_Type, size: int = SNIFF_SIZE) -> bytes: Returns an empty bytes object if the source cannot be read. """ try: - if isinstance(source, (str, Path)): + if isinstance(source, Path_Type): path = Path(source) if not path.is_file(): return b"" @@ -172,6 +172,9 @@ def read_header(source: Source_Type, size: int = SNIFF_SIZE) -> bytes: source.seek(position) return header if isinstance(header, bytes) else header.encode() except (OSError, ValueError): + # File access or read errors will fall through here, implying the reader cannot interpret the file. + # An empty byte string will be returned, but there may be other cases where this try/except are bypassed, so + # explicitly do nothing here. pass return b"" From 12eae47054806476d20c35d301d025522c25724c Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 11:54:00 -0500 Subject: [PATCH 25/25] Fix utest_import_ascii.py tests to match latest return structure after resolving conflicts --- test/data_io/utest_import_ascii.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/data_io/utest_import_ascii.py b/test/data_io/utest_import_ascii.py index 60fb84cd0..7903600d9 100644 --- a/test/data_io/utest_import_ascii.py +++ b/test/data_io/utest_import_ascii.py @@ -133,7 +133,7 @@ def test_mumag_metadata(): ) data = AsciiImporter(params=params).load_data() assert len(data) == len(filenames) - for datum in data: + for _, datum in data.items(): # Each file has 60 rows of Q, I, dI data. assert len(datum._data_contents["Q"].value) == 60 assert len(datum._data_contents["I"].value) == 60 @@ -168,7 +168,7 @@ def load_text(tmp_path, text: str, columns=Q_I_DI_COLUMNS, **kwargs) -> dict: separator_dict=ALL_SEPARATORS, **kwargs, ) - return load_data(params)[0]._data_contents + return AsciiImporter(params=params).load_data()['data.txt']._data_contents # Line numbers in the comments are the (0-based) indices used for excluded_lines.