From 68fdb28534ab62b40aedcfccece9afa7a869cd07 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 04:23:23 -0500 Subject: [PATCH 01/13] rebase conflicts --- sasdata/data_io/importers/import_ascii.py | 58 +++++++----- test/data_io/utest_import_ascii.py | 105 +++++++++++++++++++++- 2 files changed, 139 insertions(+), 24 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index b467bc191..9c0c540b9 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -1,3 +1,4 @@ +import logging import re from dataclasses import dataclass, field, replace from enum import Enum @@ -26,6 +27,8 @@ from sasdata.quantities.quantity import NamedQuantity, Quantity from sasdata.quantities.units import NamedUnit +logger = logging.getLogger(__name__) + class AsciiSeparator(Enum): Comma = (0,) @@ -121,32 +124,41 @@ def split_line(separator_dict: dict[str, bool], line: str) -> list[str]: # TODO: Implement error handling. def load_quantities(params: AsciiReaderParams, filename: str, metadata: Metadata) -> dict[str, Quantity]: - """Load a list of quantities from the filename based on the params.""" + """Load a list of quantities from the filename based on the params. + + Only the lines that will actually be used become data points. A line is skipped if it is before + ``params.starting_line``, if its (0-based) line number is in ``params.excluded_lines``, or if it can't be read as a + row of numbers (e.g. blank lines, comments, or lines with too few columns). Skipped lines are left out entirely + rather than being stored as zeros. + """ with open(filename) as ascii_file: lines = ascii_file.readlines() - arrays: list[np.ndarray] = [] - for _ in params.columns_included: - arrays.append(np.zeros(len(lines) - params.starting_line)) - for i, current_line in enumerate(lines): - if i < params.starting_line or current_line in params.excluded_lines: - continue - line_split = split_line(params.separator_dict, current_line) - try: - for j, token in enumerate(line_split): - # Sometimes in the split, there might be an extra column that doesn't need to be there (e.g. an empty - # string.) This won't convert to a float so we need to ignore it. - if j >= len(params.columns_included): - continue - # TODO: Data might not be floats. Maybe don't hard code this. - arrays[j][i - params.starting_line] = float(token) - except ValueError: - # If any of the lines contain non-numerical data, then this line can't be read in as a quantity so it - # should be ignored entirely. - print(f"Line {i + 1} skipped.") - continue + + column_count = len(params.columns_included) + rows: list[list[float]] = [] + for i, current_line in enumerate(lines): + if i < params.starting_line or i in params.excluded_lines: + continue + line_split = split_line(params.separator_dict, current_line) + # Sometimes in the split, there might be extra columns that don't need to be there (e.g. an empty string.) + # These won't convert to a float so we need to ignore them. + tokens = line_split[:column_count] + if len(tokens) < column_count: + logger.info(f"{path.basename(filename)}: line {i + 1} skipped as it has too few columns.") + continue + try: + # TODO: Values are parsed as floats; decimal commas, Fortran 'D' exponents and non-numeric columns (e.g. a + # boolean mask) are not handled and cause the row to be skipped. + rows.append([float(token) for token in tokens]) + except ValueError: + # If any of the lines contain non-numerical data, then this line can't be read in as a quantity so it + # should be ignored entirely. + logger.info(f"{path.basename(filename)}: line {i + 1} skipped as it contains non-numerical data.") + + data = np.array(rows, dtype=float).reshape(len(rows), column_count) file_quantities = { - name: NamedQuantity(name, arrays[i], unit, id_header=metadata.id_header) - for i, (name, unit) in enumerate(params.columns_included) + name: NamedQuantity(name, data[:, j].copy(), unit, id_header=metadata.id_header) + for j, (name, unit) in enumerate(params.columns_included) } return file_quantities diff --git a/test/data_io/utest_import_ascii.py b/test/data_io/utest_import_ascii.py index 12fe77edb..1e6fdaa48 100644 --- a/test/data_io/utest_import_ascii.py +++ b/test/data_io/utest_import_ascii.py @@ -127,14 +127,19 @@ def test_mumag_metadata(): params = AsciiReaderParams( filenames=param_filenames, columns=[(column, per_angstrom) for column in guess_columns(3, one_dim)], - separator_dict={"Comma": True}, + # The MuMag files are tab separated. + separator_dict={"Comma": True, "Tab": True}, metadata=metadata, ) data = AsciiImporter(params=params).load_data() assert len(data) == len(filenames) for datum in data.values(): + # Each file has 60 rows of Q, I, dI data. + assert len(datum._data_contents["Q"].value) == 60 + assert len(datum._data_contents["I"].value) == 60 match datum.name: case "1_33_1640_22.874115.csv": + assert datum._data_contents["Q"].value[0] == pytest.approx(0.02732) assert datum.metadata.raw.filter("counting_index") == ["1"] assert datum.metadata.raw.filter("applied_magnetic_field") == ["33"] assert datum.metadata.raw.filter("saturation_magnetization") == ["1640"] @@ -144,3 +149,101 @@ def test_mumag_metadata(): assert datum.metadata.raw.filter("applied_magnetic_field") == ["1270"] assert datum.metadata.raw.filter("saturation_magnetization") == ["1640"] assert datum.metadata.raw.filter("demagnetizing_field") == ["24"] + + +# Tests for excluded lines, and for lines that are skipped when loading (issue #261). + +ALL_SEPARATORS = {"Comma": True, "Whitespace": True, "Tab": True} +Q_I_DI_COLUMNS = [("Q", per_angstrom), ("I", per_centimeter), ("dI", per_centimeter)] + + +def load_text(tmp_path, text: str, **kwargs) -> dict: + """Write text to a temporary ASCII file, load it with Q, I, dI columns, and return the loaded quantities.""" + filename = tmp_path / "data.txt" + filename.write_text(text) + params = AsciiReaderParams( + filenames=[str(filename)], + columns=Q_I_DI_COLUMNS, + separator_dict=ALL_SEPARATORS, + **kwargs, + ) + return load_data(params)[0]._data_contents + + +# Line numbers in the comments are the (0-based) indices used for excluded_lines. +HEADER_AND_DATA = ( + "Q I dI\n" # 0 + "0.01 100 1\n" # 1 + "0.02 90 2\n" # 2 + "0.03 80 3\n" # 3 + "0.04 70 4\n" # 4 +) + + +def test_no_excluded_lines(tmp_path): + data = load_text(tmp_path, HEADER_AND_DATA, starting_line=1) + assert data["Q"].value == pytest.approx([0.01, 0.02, 0.03, 0.04]) + assert data["I"].value == pytest.approx([100, 90, 80, 70]) + + +def test_excluded_line_is_removed(tmp_path): + data = load_text(tmp_path, HEADER_AND_DATA, starting_line=1, excluded_lines={2}) + # The excluded row must be removed, not replaced by zeros. + assert data["Q"].value == pytest.approx([0.01, 0.03, 0.04]) + assert data["I"].value == pytest.approx([100, 80, 70]) + + +def test_multiple_excluded_lines(tmp_path): + data = load_text(tmp_path, HEADER_AND_DATA, starting_line=1, excluded_lines={1, 4}) + assert data["Q"].value == pytest.approx([0.02, 0.03]) + assert data["I"].value == pytest.approx([90, 80]) + + +def test_all_data_lines_excluded(tmp_path): + data = load_text(tmp_path, HEADER_AND_DATA, starting_line=1, excluded_lines={1, 2, 3, 4}) + assert len(data["Q"].value) == 0 + assert len(data["I"].value) == 0 + + +def test_excluded_lines_before_starting_line_are_ignored(tmp_path): + # Excluding a header line (which is already skipped by the starting line) must not affect the data. + data = load_text(tmp_path, HEADER_AND_DATA, starting_line=1, excluded_lines={0}) + assert data["Q"].value == pytest.approx([0.01, 0.02, 0.03, 0.04]) + + +def test_excluded_line_numbers_count_from_start_of_file(tmp_path): + # Line numbers in excluded_lines refer to the whole file, not to the position after the starting line. + text = "# comment\n# comment\nQ I dI\n0.01 100 1\n0.02 90 2\n0.03 80 3\n" + data = load_text(tmp_path, text, starting_line=3, excluded_lines={4}) + assert data["Q"].value == pytest.approx([0.01, 0.03]) + + +def test_uncertainties_stay_aligned_with_excluded_lines(tmp_path): + # dI is merged into I as its uncertainty, so it must lose the same rows as I. + data = load_text(tmp_path, HEADER_AND_DATA, starting_line=1, excluded_lines={2}) + _, standard_error = data["I"].in_units_of_with_standard_error(per_centimeter) + assert standard_error == pytest.approx([1, 3, 4]) + + +def test_unreadable_lines_are_skipped_not_zeroed(tmp_path): + text = ( + "Q I dI\n" + "0.01 100 1\n" + "\n" # blank line + "not a data line\n" + "0.02 90 2\n" + "0.03 80\n" # too few columns + "0.04 70 4\n" + "\n" # trailing blank line + ) + data = load_text(tmp_path, text, starting_line=1) + assert data["Q"].value == pytest.approx([0.01, 0.02, 0.04]) + assert data["I"].value == pytest.approx([100, 90, 70]) + assert 0 not in data["Q"].value + + +def test_extra_columns_are_ignored(tmp_path): + text = "0.01 100 1 0.001 5\n0.02 90 2 0.002 5\n" + data = load_text(tmp_path, text) + assert data["Q"].value == pytest.approx([0.01, 0.02]) + assert data["I"].value == pytest.approx([100, 90]) From b28455239f7bc5ff9dcc60da67bbe66fb2289d47 Mon Sep 17 00:00:00 2001 From: gonzalezma Date: Sun, 8 Nov 2026 14:39:06 +0100 Subject: [PATCH 02/13] Read ASCII columns by position so ignored columns are correctly handled (#263) - Read each loaded column from its own position; ignored columns are skipped and their values aren't parsed - Add AsciiReaderParams.column_positions_included - guess_params_from_filename keeps columns with no known unit as instead of dropping them (e.g. dPhi in 4-column I vs Phi files) - Add tests Fixes #263 --- sasdata/data_io/importers/import_ascii.py | 35 ++++++++++++++--------- test/data_io/utest_import_ascii.py | 30 +++++++++++++++++-- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index 9c0c540b9..c24bbb50a 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -68,13 +68,19 @@ def initialise_metadata(self): self.metadata.filename_separator[basename] = "_" self.metadata.filename_specific_metadata[basename] = {} + @staticmethod + def is_column_included(column: tuple[str, NamedUnit | None]) -> bool: + """Whether a column is loaded, i.e. it isn't ignored and has a unit.""" + return column[0] != "" and isinstance(column[1], NamedUnit) + @property def columns_included(self) -> list[tuple[str, NamedUnit]]: - return [ - column - for column in self.columns - if column[0] != "" and isinstance(column[1], NamedUnit) - ] + return [column for column in self.columns if self.is_column_included(column)] + + @property + def column_positions_included(self) -> list[int]: + """Position in each line of the columns in columns_included. Ignored columns still take up a position.""" + return [position for position, column in enumerate(self.columns) if self.is_column_included(column)] def guess_params_from_filename(filename: str, dataset_type: DatasetType) -> AsciiReaderParams: @@ -85,10 +91,11 @@ def guess_params_from_filename(filename: str, dataset_type: DatasetType) -> Asci lines_split = [split_line(separator_dict, line) for line in lines] startpos = guess_starting_position(lines_split) colcount = guess_column_count(lines_split, startpos) + # Columns we don't know the unit for are ignored. They are kept as rather than removed, so the + # columns after them still line up with the data. columns = [ - (x, get_default_unit(x, unit_kinds[x])) + (x, get_default_unit(x, unit_kinds[x])) if x in unit_kinds else ("", None) for x in guess_columns(colcount, dataset_type) - if x in unit_kinds ] params = AsciiReaderParams( [filename], @@ -134,22 +141,24 @@ def load_quantities(params: AsciiReaderParams, filename: str, metadata: Metadata with open(filename) as ascii_file: lines = ascii_file.readlines() - column_count = len(params.columns_included) + # Ignored columns are skipped, but still count when finding the position of the other columns in each line. + positions = params.column_positions_included + column_count = len(positions) + required_length = max(positions) + 1 if positions else 0 rows: list[list[float]] = [] for i, current_line in enumerate(lines): if i < params.starting_line or i in params.excluded_lines: continue line_split = split_line(params.separator_dict, current_line) - # Sometimes in the split, there might be extra columns that don't need to be there (e.g. an empty string.) - # These won't convert to a float so we need to ignore them. - tokens = line_split[:column_count] - if len(tokens) < column_count: + # Values after the last loaded column (e.g. an empty string at the end of the line, or extra columns) and in + # ignored columns aren't read, so they don't need to be numbers. + if len(line_split) < required_length: logger.info(f"{path.basename(filename)}: line {i + 1} skipped as it has too few columns.") continue try: # TODO: Values are parsed as floats; decimal commas, Fortran 'D' exponents and non-numeric columns (e.g. a # boolean mask) are not handled and cause the row to be skipped. - rows.append([float(token) for token in tokens]) + rows.append([float(line_split[position]) for position in positions]) except ValueError: # If any of the lines contain non-numerical data, then this line can't be read in as a quantity so it # should be ignored entirely. diff --git a/test/data_io/utest_import_ascii.py b/test/data_io/utest_import_ascii.py index 1e6fdaa48..1fa989443 100644 --- a/test/data_io/utest_import_ascii.py +++ b/test/data_io/utest_import_ascii.py @@ -157,13 +157,14 @@ def test_mumag_metadata(): Q_I_DI_COLUMNS = [("Q", per_angstrom), ("I", per_centimeter), ("dI", per_centimeter)] -def load_text(tmp_path, text: str, **kwargs) -> dict: - """Write text to a temporary ASCII file, load it with Q, I, dI columns, and return the loaded quantities.""" +def load_text(tmp_path, text: str, columns=Q_I_DI_COLUMNS, **kwargs) -> dict: + """Write text to a temporary ASCII file, load it with the given columns (Q, I, dI by default), and return the + loaded quantities.""" filename = tmp_path / "data.txt" filename.write_text(text) params = AsciiReaderParams( filenames=[str(filename)], - columns=Q_I_DI_COLUMNS, + columns=columns, separator_dict=ALL_SEPARATORS, **kwargs, ) @@ -247,3 +248,26 @@ def test_extra_columns_are_ignored(tmp_path): data = load_text(tmp_path, text) assert data["Q"].value == pytest.approx([0.01, 0.02]) assert data["I"].value == pytest.approx([100, 90]) + + +# Tests for ignored columns (issue #263). + +IGNORE = ("", None) + + +def test_ignored_column_in_the_middle(tmp_path): + columns = [("Q", per_angstrom), IGNORE, ("I", per_centimeter), ("dI", per_centimeter)] + data = load_text(tmp_path, "0.01 999 100 1\n0.02 999 90 2\n", columns=columns) + assert data["Q"].value == pytest.approx([0.01, 0.02]) + assert data["I"].value == pytest.approx([100, 90]) + _, standard_error = data["I"].in_units_of_with_standard_error(per_centimeter) + assert standard_error == pytest.approx([1, 2]) + + +def test_column_without_unit_is_ignored(tmp_path): + # A column with no unit isn't loaded, but still takes up its position in the line. + columns = [("Q", per_angstrom), ("dQ", None), ("I", per_centimeter), ("dI", per_centimeter)] + data = load_text(tmp_path, "0.01 999 100 1\n0.02 999 90 2\n", columns=columns) + assert "dQ" not in data + assert data["Q"].value == pytest.approx([0.01, 0.02]) + assert data["I"].value == pytest.approx([100, 90]) From b89c520f7735b73637f1540672d801b67bc831db Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 03:51:30 -0500 Subject: [PATCH 03/13] Put the exclude_patterns in the correct place --- sasdata/data_io/exporter.py | 292 ++++++++++++++++++++++++++++++++++-- 1 file changed, 281 insertions(+), 11 deletions(-) diff --git a/sasdata/data_io/exporter.py b/sasdata/data_io/exporter.py index 26a5165d7..dee77685d 100644 --- a/sasdata/data_io/exporter.py +++ b/sasdata/data_io/exporter.py @@ -1,21 +1,291 @@ +""" +Base class and dispatch for all sasdata exporters. + +Every module in :mod:`sasdata.data_io.exporters` defines a child of :class:`Exporter`. Each child knows how to write a +list of SasMeasurement objects to a single file in one format, in a form that the matching importer in +:mod:`sasdata.data_io.importers` (and SasView) can read back in. + +Children register themselves automatically when they are defined. :meth:`Exporter.export_data` selects the exporter +for each file, either by name or from the file extension, checks that the destination can be written to, and then +writes the file:: + + from sasdata.data_io.exporter import Exporter + + errors = Exporter().export_data({"out.h5": [data_1, data_2], "out.csv": data_1}) + errors = Exporter().export_data({"no_extension": data_1}, exporter="canSAS XML") +""" + +import importlib +import logging +import os +import tempfile +from collections.abc import Iterable from pathlib import Path +from typing import ClassVar -from sasdata.data import SasData +from sasdata.data import SasData, SasMeasurement +from sasdata.dataset_types import DatasetType + +logger = logging.getLogger(__name__) Path_Type = str | Path +Value_Type = str | int | float | bool | list[str] | list[int] | list[float] +Data_Type = SasData | Iterable[SasData] | dict[str, SasData] + + +class ExportError(Exception): + """Raised when data cannot be exported.""" class Exporter: - def __init__(self): - pass + """ + Base class for all data exporters. + + Child classes must implement :meth:`write` and should set :attr:`name`, :attr:`extensions` and + :attr:`supported_dataset_types`. Instantiating ``Exporter`` directly gives a generic exporter whose + :meth:`export_data` selects the correct child exporter for each file. + """ + + # Human-readable name of the format written by the exporter. Used to select the exporter by name. + name: ClassVar[str] = "Generic" + # Alternative names that can be used to select the exporter. + aliases: ClassVar[tuple[str, ...]] = () + # File extensions (lower case, including the leading '.') associated with the format. The first is the default. + extensions: ClassVar[tuple[str, ...]] = () + # Dataset types that can be written to, and read back from, the format. An empty tuple means any type. + supported_dataset_types: ClassVar[tuple[DatasetType, ...]] = () + # Whether more than one dataset can be written to a single file. + multiple_datasets: ClassVar[bool] = True + # All concrete exporters, populated automatically by :meth:`__init_subclass__`. + _registry: ClassVar[list[type["Exporter"]]] = [] + + def __init_subclass__(cls, register: bool = True, **kwargs): + super().__init_subclass__(**kwargs) + if register and cls not in Exporter._registry: + Exporter._registry.append(cls) + + def __init__(self, config: dict[str, Value_Type] | None = None): + self.config: dict[str, Value_Type] = config if config is not None else {} + + # ------------------------------------------------------------------------------------------------------------------ + # Interface to be implemented by every child exporter + # ------------------------------------------------------------------------------------------------------------------ + + def write(self, data: dict[str, SasMeasurement], path: Path): + """ + Write ``data`` to ``path``. The data and destination have already been validated when this is called. + + :param data: The measurements to write, keyed by a unique name. Formats that can hold several datasets use the + keys to name them, so the same keys are returned when the file is imported. + :param path: The file to write. This is a temporary file in the destination directory, which is moved to the + final destination once it has been written successfully. + """ + raise NotImplementedError(f"{type(self).__name__} does not implement write()") + + # ------------------------------------------------------------------------------------------------------------------ + # Exporter selection + # ------------------------------------------------------------------------------------------------------------------ + + @staticmethod + def exporters() -> list[type["Exporter"]]: + """All registered exporters.""" + # Importing the package imports every exporter module, which registers the child classes. + importlib.import_module("sasdata.data_io.exporters") + return list(Exporter._registry) + + @classmethod + def find_exporter(cls, path: Path_Type | None = None, name: str | type["Exporter"] | None = None) \ + -> type["Exporter"] | None: + """ + Select an exporter by name (case-insensitive, checking :attr:`name` and :attr:`aliases`), or if no name is + given, from the extension of ``path``. + """ + exporters = cls.exporters() + if isinstance(name, type): + return name if issubclass(name, Exporter) and name in exporters else None + if name is not None: + wanted = name.strip().lower() + for exporter in exporters: + if wanted in (n.lower() for n in (exporter.name, *exporter.aliases)): + return exporter + return None + if path is not None: + extension = Path(path).suffix.lower() + for exporter in exporters: + if extension in exporter.extensions: + return exporter + return None + + # ------------------------------------------------------------------------------------------------------------------ + # Validation + # ------------------------------------------------------------------------------------------------------------------ - def export_data(self, file_map: dict[Path_Type, list[SasData]]) -> list[str]: - errors = [] - for path, sas_data_list in file_map: - file = Path(path) + @staticmethod + def validate_destination(path: Path_Type, overwrite: bool = True) -> Path: + """ + Check that ``path`` can be written to: its directory must exist, be a directory, and be writable, and if the + file already exists it must be a writable file and ``overwrite`` must be True. + + :return: The absolute path to the file. + :raises FileNotFoundError, NotADirectoryError, IsADirectoryError, FileExistsError, PermissionError: + """ + file = Path(path).expanduser().absolute() + directory = file.parent + if not directory.exists(): + raise FileNotFoundError(f'The directory "{directory}" does not exist.') + if not directory.is_dir(): + raise NotADirectoryError(f'"{directory}" is not a directory.') + if file.is_dir(): + raise IsADirectoryError(f'"{file}" is a directory.') + if not os.access(directory, os.W_OK | os.X_OK): + raise PermissionError(f'You do not have permission to write to the directory "{directory}".') + if file.exists(): + if not overwrite: + raise FileExistsError(f'"{file}" already exists.') + if not os.access(file, os.W_OK): + raise PermissionError(f'You do not have permission to overwrite "{file}".') + # os.access does not reflect every permission system (e.g. Windows ACLs), so confirm by creating a file. + try: + handle, probe = tempfile.mkstemp(dir=directory, prefix=".sasdata_", suffix=".tmp") + except OSError as e: + raise PermissionError(f'You do not have permission to write to the directory "{directory}": {e}') from e + os.close(handle) + os.remove(probe) + return file + + def validate_data(self, data: list[SasData]): + """ + Check the data can be written by this exporter. + + :raises ExportError: If there is no data, too many datasets, or a dataset type the format cannot hold. + """ + if len(data) == 0: + raise ExportError("There is no data to export.") + if len(data) > 1 and not self.multiple_datasets: + raise ExportError(f"The {self.name} format can only hold one dataset per file, but {len(data)} were given.") + for datum in data: + if not isinstance(datum, SasMeasurement): + raise ExportError(f'"{datum.name}" is a {type(datum).__name__}, which cannot be exported yet.') + if self.supported_dataset_types and datum.dataset_type not in self.supported_dataset_types: + supported = ", ".join(t.name for t in self.supported_dataset_types) + raise ExportError(f'"{datum.name}" is {datum.dataset_type.name} data, but the {self.name} format ' + f'only supports: {supported}.') + + # ------------------------------------------------------------------------------------------------------------------ + # Export + # ------------------------------------------------------------------------------------------------------------------ + + def export_data(self, file_map: dict[Path_Type, Data_Type], exporter: str | type["Exporter"] | None = None, + overwrite: bool = True) -> list[str]: + """ + Export data to one or more files. + + :param file_map: Map of each destination file to the data to write to it. The data may be a single dataset, + a list of datasets, or a dictionary of datasets as returned by the importers. + :param exporter: The name (or class) of the exporter to use for every file. If None and this is a child + exporter, that exporter is used; otherwise, the exporter is chosen from each file extension. + :param overwrite: Whether existing files may be replaced. + :return: A list of error messages for anything that could not be exported. An empty list means success. + """ + errors: list[str] = [] + for path, data in file_map.items(): + names = "data" try: - with open(file, 'w') as export_file: - pass - except Exception: - errors.append(f'Unable to write {', '.join([data.name for data in sas_data_list])} to "{file}".') + data_map = keyed_data(data) + names = ", ".join(data_map) or names + exporter_class = self._select_exporter(path, exporter) + file = self.validate_destination(path, overwrite) + instance = self if type(self) is exporter_class else exporter_class(self.config) + instance.validate_data(list(data_map.values())) + instance._write_safely(data_map, file) + except Exception as e: + logger.debug("Export failed", exc_info=True) + errors.append(f'Unable to export {names} to "{path}": {e}') return errors + + @classmethod + def export(cls, data: Data_Type, path: Path_Type, exporter: str | type["Exporter"] | None = None, + config: dict[str, Value_Type] | None = None, overwrite: bool = True): + """ + Convenience method: export ``data`` to a single file. + + :raises ExportError: If the data could not be exported. + """ + instance = cls(config) + errors = instance.export_data({path: data}, exporter=exporter, overwrite=overwrite) + if errors: + raise ExportError("\n".join(errors)) + + def _select_exporter(self, path: Path_Type, exporter: str | type["Exporter"] | None) -> type["Exporter"]: + if exporter is not None: + selected = self.find_exporter(name=exporter) + if selected is None: + known = ", ".join(e.name for e in self.exporters()) + raise ExportError(f'Unknown exporter "{exporter}". Known exporters are: {known}.') + return selected + if type(self) is not Exporter: + return type(self) + selected = self.find_exporter(path=path) + if selected is None: + raise ExportError(f'Unable to determine the export format from the extension of "{path}". ' + f'Choose an exporter or use one of: {", ".join(all_extensions())}.') + return selected + + def _write_safely(self, data: dict[str, SasMeasurement], file: Path): + """ + Write to a temporary file in the destination directory then move it into place, so an existing file is never + left half-written if the export fails. + """ + handle, temp_name = tempfile.mkstemp(dir=file.parent, prefix=f".{file.name}.", suffix=".tmp") + os.close(handle) + temp_file = Path(temp_name) + try: + self.write(data, temp_file) + # mkstemp creates files readable only by the owner; give the export the permissions a new file would have + # (or keep those of the file being replaced). + os.chmod(temp_file, file.stat().st_mode if file.exists() else default_file_mode()) + os.replace(temp_file, file) + finally: + if temp_file.exists(): + temp_file.unlink() + + +# ---------------------------------------------------------------------------------------------------------------------- +# Helpers shared by the exporters +# ---------------------------------------------------------------------------------------------------------------------- + +def keyed_data(data: Data_Type) -> dict[str, SasData]: + """ + Convert a single dataset, an iterable of datasets or a dict of datasets into a dict of datasets with unique keys. + The keys of a dict are kept; otherwise, the datasets are keyed by name. + """ + if isinstance(data, dict): + return {str(key): value for key, value in data.items()} + data_list = [data] if isinstance(data, SasData) else list(data) + return dict(zip(unique_names(data_list), data_list)) + + +def default_file_mode() -> int: + """The permissions given to a newly created file, taking the umask into account.""" + umask = os.umask(0) + os.umask(umask) + return 0o666 & ~umask + + +def all_extensions() -> list[str]: + """Every file extension that an exporter is registered for.""" + return [extension for exporter in Exporter.exporters() for extension in exporter.extensions] + + +def unique_names(data: list[SasData]) -> list[str]: + """The names of the datasets, with a numeric suffix added to any repeated names.""" + names: list[str] = [] + for index, datum in enumerate(data): + name = str(getattr(datum, "name", "") or f"SasData{index + 1:02d}") + candidate = name + suffix = 2 + while candidate in names: + candidate = f"{name} ({suffix})" + suffix += 1 + names.append(candidate) + return names From beeb52ad2b9845c02a1aa3a41fe026b25850b649 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 04:10:29 -0500 Subject: [PATCH 04/13] Create base ascii and csv exporters --- sasdata/data_io/exporters/export_ascii.py | 153 ++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 sasdata/data_io/exporters/export_ascii.py diff --git a/sasdata/data_io/exporters/export_ascii.py b/sasdata/data_io/exporters/export_ascii.py new file mode 100644 index 000000000..02d5f63c4 --- /dev/null +++ b/sasdata/data_io/exporters/export_ascii.py @@ -0,0 +1,153 @@ +""" +Export SasData to delimited text files: whitespace separated multi-column ASCII files and comma separated (CSV) files. + +The files start with comment lines describing the data, followed by a line naming each column and its units, e.g. +``Q(Ang^-1) I(cm^-1) dI(cm^-1) dQ(Ang^-1)``, so :class:`sasdata.data_io.importers.import_ascii.AsciiImporter` reads the +columns back with the correct names and units. + +The columns are written in the order SasView expects, in the units SasView assumes (1/Å and 1/cm): + +* 1D data: ``Q I dI dQ``, then any of ``dQw dQl Qmean Shadowfactor``. SasView reads these from ``.txt`` and ``.csv`` + files. +* 2D data: ``Qx Qy I dI Qz dQx dQy ShadowFactor``, then any of ``dQz mask``, using the NIST/IGOR 2D header that SasView + reads from ``.dat`` files. + +Columns that are needed to keep the positions of later columns, but have no data, are written as zeros and named +```` so they are skipped when the file is imported. +""" + +from pathlib import Path +from typing import ClassVar + +import numpy as np + +from sasdata.data import SasMeasurement +from sasdata.data_io.exporter import Exporter +from sasdata.dataset_types import one_dim, three_dim, two_dim +from sasdata.quantities.units import Unit + +IGNORE = "" + +# Columns written for each dataset type. The names are those of the quantities in the data, apart from errors, which +# are written as 'd' + the name of the quantity. The first group are the columns SasView reads by position, and so +# must be written in this order; the second group are optional extras, written after them when present. +COLUMN_ORDERS: dict[str, tuple[list[str], list[str]]] = { + one_dim.name: (["Q", "I", "dI", "dQ"], ["dQw", "dQl", "Qmean", "Shadowfactor"]), + two_dim.name: (["Qx", "Qy", "I", "dI", "Qz", "dQx", "dQy", "ShadowFactor"], ["dQz", "mask"]), + three_dim.name: (["Qx", "Qy", "I", "dI", "Qz", "dQx", "dQy", "ShadowFactor"], ["dQz", "mask"]), +} + +# The columns that must always be written, even if they are placeholders, for SasView to read the file correctly. +REQUIRED_COLUMNS = { + one_dim.name: 2, + two_dim.name: 3, + three_dim.name: 5, +} + +def format_value(value) -> str: + return repr(float(value)) + + +class DelimitedTextExporter(Exporter, register=False): + """Shared implementation of the ASCII and CSV exporters. Not registered as an exporter itself.""" + + # The string placed between values on each line. + delimiter: ClassVar[str] = " " + multiple_datasets = False + + def write(self, data: dict[str, SasMeasurement], path: Path): + datum = next(iter(data.values())) + names, columns = self.build_columns(datum) + with open(path, "w", encoding="utf-8", newline="\n") as f: + for line in self.header_lines(datum, names): + f.write(line + "\n") + for row in zip(*columns): + f.write(self.delimiter.join(format_value(v) for v in row) + "\n") + + def header_lines(self, datum: SasMeasurement, names: list[str]) -> list[str]: + """Comment lines describing the data, finishing with the line naming the columns.""" + metadata = datum.metadata + lines = [ + f"# {datum.name}", + f"# Exported by sasdata as {datum.dataset_type.name} data", + ] + if metadata.title: + lines.append(f"# Title: {metadata.title}") + if metadata.run: + lines.append(f"# Run: {' '.join(metadata.run)}") + lines.append(self.delimiter.join(names)) + return lines + + def build_columns(self, datum: SasMeasurement) -> tuple[list[str], list[np.ndarray]]: + """ + Find the columns to write. Returns the column headings (name and units) and the values of each column, in the + units given in the headings. + """ + contents = {name: quantity for name, quantity in datum.data_contents.items()} + positional, extras = COLUMN_ORDERS[datum.dataset_type.name] + available: dict[str, tuple[np.ndarray, Unit]] = {} + for name, quantity in contents.items(): + available[name] = (np.asarray(quantity.value, dtype=float), quantity.units) + if quantity.has_error and f"d{name}" not in contents: + available[f"d{name}"] = (np.asarray(quantity.standard_error.value, dtype=float), quantity.units) + + # Write the positional columns up to the last one with data, filling any gaps with placeholders. + present = [index for index, name in enumerate(positional) if name in available] + last = max(present + [REQUIRED_COLUMNS[datum.dataset_type.name] - 1]) + extra_names = [name for name in extras if name in available] + if extra_names: + last = len(positional) - 1 + size = np.asarray(contents["I"].value).size + + headings: list[str] = [] + columns: list[np.ndarray] = [] + for name in positional[:last + 1] + extra_names: + if name in available: + values, unit = available[name] + headings.append(f"{name}({unit.ascii_symbol})") + columns.append(np.asarray(values, dtype=float).reshape(-1)) + else: + headings.append(IGNORE) + columns.append(np.zeros(size)) + return headings, columns + + +class AsciiExporter(DelimitedTextExporter): + """ + Exporter for whitespace separated multi-column ASCII files. Supports 1D and 2D data, one dataset per file. + + SasView chooses how to read text files from their extension, so 1D data should be saved as ``.txt`` and 2D data as + ``.dat`` for SasView to read it back. The sasdata importer reads either from the column headings. + """ + + name = "ASCII" + aliases = ("text", "multi-column ascii", "txt", "dat") + extensions = (".txt", ".dat", ".abs") + supported_dataset_types = (one_dim, two_dim, three_dim) + delimiter = " " + + def header_lines(self, datum: SasMeasurement, names: list[str]) -> list[str]: + lines = super().header_lines(datum, names) + if datum.dataset_type != one_dim: + # The NIST/IGOR 2D header used by SasView's 2D ASCII reader, which needs to know whether the fourth column + # is the uncertainty in I, and that the data starts on the line after 'ASCII data'. + descriptions = {"Qx": "Qx", "Qy": "Qy", "I": "I(Qx,Qy)", "dI": "err(I)", "Qz": "Qz", + "dQx": "SigmaQ_parall", "dQy": "SigmaQ_perp", "ShadowFactor": "fSubS(beam stop shadow)"} + column_names = [n.split("(")[0] for n in names] + if column_names[3:4] == [IGNORE]: + # No uncertainties: describe the column as something other than err(I). + descriptions["dI"] = "unused" + described = [descriptions.get(n, n) for n in column_names] + lines.insert(-1, "Data columns are " + " - ".join(described)) + lines.append("ASCII data") + return lines + + +class CsvExporter(DelimitedTextExporter): + """Exporter for comma separated (CSV) files. Supports 1D data, one dataset per file.""" + + name = "CSV" + aliases = ("comma separated values", "comma-separated values") + extensions = (".csv",) + supported_dataset_types = (one_dim,) + delimiter = "," From 8fea03b890bae438932590ec301b0360d8c7a93b Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 04:12:36 -0500 Subject: [PATCH 05/13] Create base cansas xml exporter --- sasdata/data_io/exporters/export_nxcansas.py | 170 +++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 sasdata/data_io/exporters/export_nxcansas.py diff --git a/sasdata/data_io/exporters/export_nxcansas.py b/sasdata/data_io/exporters/export_nxcansas.py new file mode 100644 index 000000000..378ee7cc6 --- /dev/null +++ b/sasdata/data_io/exporters/export_nxcansas.py @@ -0,0 +1,170 @@ +""" +Export SasData to NXcanSAS (HDF5) files. + +The files follow the NXcanSAS definition (https://manual.nexusformat.org/classes/applications/NXcanSAS.html) as +written by SasView: each dataset is written to its own SASentry, with its data in a SASdata group. Uncertainties are +linked to their data with the ``uncertainties`` attribute and resolutions to Q with the ``resolutions`` attribute, so +the files can be read by both :class:`sasdata.data_io.importers.import_hdf5.HDF5Importer` and SasView. +""" + +import re +from pathlib import Path + +import h5py +import numpy as np + +from sasdata.data import SasMeasurement +from sasdata.data_io.exporter import Exporter, data_contents +from sasdata.dataset_types import one_dim, three_dim, two_dim +from sasdata.quantities.quantity import Quantity + +# canSAS and NeXus classes of the groups written by Metadata.as_h5, keyed by a pattern matching the group name. +GROUP_CLASSES: dict[str, tuple[str, str]] = { + r"sasentry\d*": ("SASentry", "NXentry"), + r"sasdata\d*": ("SASdata", "NXdata"), + r"sassample": ("SASsample", "NXsample"), + r"sasinstrument": ("SASinstrument", "NXinstrument"), + r"sassource": ("SASsource", "NXsource"), + r"sascollimation\d*": ("SAScollimation", "NXcollimator"), + r"sasaperture\d*": ("SASaperture", "NXaperture"), + r"sasdetector\d*": ("SASdetector", "NXdetector"), + r"sasprocess\d*": ("SASprocess", "NXprocess"), + r"sasnote\d*": ("SASnote", "NXnote"), +} + +# The abscissae of each dataset type, in the order expected by the I_axes attribute. +AXES = { + one_dim.name: ["Q"], + two_dim.name: ["Qx", "Qy"], + three_dim.name: ["Qx", "Qy", "Qz"], +} + +# The resolutions of each axis, in the order they are listed in the resolutions attribute. +RESOLUTIONS = { + "Q": ["dQ", "dQw", "dQl"], + "Qx": ["dQx"], + "Qy": ["dQy"], + "Qz": ["dQz"], +} + + +class NXcanSASExporter(Exporter): + """Exporter for NXcanSAS HDF5 files. Supports 1D, 2D and 3D data, with any number of datasets per file.""" + + name = "NXcanSAS" + aliases = ("NXcanSAS HDF5", "HDF5", "h5", "nexus") + extensions = (".h5", ".hdf5", ".hdf", ".nxs") + supported_dataset_types = (one_dim, two_dim, three_dim) + multiple_datasets = True + + def write(self, data: dict[str, SasMeasurement], path: Path): + with h5py.File(path, "w") as f: + f.attrs["NX_class"] = "NXroot" + f.attrs["creator"] = "sasdata" + for index, (key, datum) in enumerate(data.items()): + entry = f.create_group(f"sasentry{index + 1:02d}") + self.write_entry(entry, key, datum) + f.attrs["default"] = "sasentry01" + + def write_entry(self, entry: h5py.Group, key: str, datum: SasMeasurement): + """Write one dataset, and its metadata, to a SASentry group.""" + entry.attrs["version"] = "1.1" + entry.attrs["name"] = datum.name + if not key.startswith("sasentry"): + # The importer keys each entry by this attribute, or by the group name if it is absent. + entry.attrs["sasview_key"] = key + entry.attrs["default"] = "sasdata01" + datum.metadata.as_h5(entry) + if "definition" not in entry: + entry.create_dataset("definition", data=["NXcanSAS"]) + if not any(name.startswith("run") for name in entry): + # A run is required by the NXcanSAS definition. + entry.create_dataset("run", data=[""]) + self.write_data(entry.create_group("sasdata01"), datum) + add_classes(entry) + + @staticmethod + def write_data(group: h5py.Group, datum: SasMeasurement): + """Write the quantities of a dataset to a SASdata group.""" + contents = data_contents(datum) + axes = AXES[datum.dataset_type.name] + shape = grid_shape(contents, axes) + + group.attrs["signal"] = "I" + group.attrs["I_axes"] = ",".join(axes) + group.attrs["Q_indices"] = np.arange(len(axes), dtype=np.int32) + for axis_index, axis in enumerate(axes): + group.attrs[f"{axis}_indices"] = np.int32(axis_index) + + for name, quantity in contents.items(): + values = reshape(quantity.value, shape) + dataset = write_quantity(group, name, values, quantity) + if quantity.has_error: + error_name = f"{name}dev" + write_quantity(group, error_name, reshape(quantity.standard_error.value, shape), quantity) + dataset.attrs["uncertainties"] = error_name + resolutions = [r for r in RESOLUTIONS.get(name, []) if r in contents] + if len(resolutions) == 1: + dataset.attrs["resolutions"] = resolutions[0] + elif resolutions: + # Multiple resolutions (e.g. slit smearing) are stored as an array of names. + dataset.attrs["resolutions"] = np.array(resolutions, dtype=h5py.string_dtype()) + + if "mask" in contents: + group.attrs["mask"] = "mask" + + +def write_quantity(group: h5py.Group, name: str, values, quantity: Quantity) -> h5py.Dataset: + """Write values to a dataset with the units of the quantity.""" + dataset = group.create_dataset(name, data=np.asarray(values)) + dataset.attrs["units"] = quantity.units.ascii_symbol + return dataset + + +def add_classes(group: h5py.Group): + """Add the canSAS_class and NX_class attributes to every recognised group below (and including) ``group``.""" + name = group.name.split("/")[-1] + for pattern, (cansas_class, nx_class) in GROUP_CLASSES.items(): + if re.fullmatch(pattern, name): + group.attrs["canSAS_class"] = cansas_class + group.attrs["NX_class"] = nx_class + break + for child in group.values(): + if isinstance(child, h5py.Group): + add_classes(child) + + +def grid_shape(contents: dict[str, Quantity], axes: list[str]) -> tuple[int, ...] | None: + """ + For 2D data stored as flat lists of points, find the detector grid the points lie on, so the data can be written + as 2D arrays (which SasView requires to recognise 2D NXcanSAS data). Returns None if the data is already gridded, + is not 2D, or the points do not lie on a regular grid. + """ + if axes != ["Qx", "Qy"]: + return None + qx = np.asarray(contents["Qx"].value) + qy = np.asarray(contents["Qy"].value) + if qx.ndim != 1 or qy.shape != qx.shape or qx.size < 4: + return None + for fast, slow in ((qx, qy), (qy, qx)): + # The number of points before the slow axis changes value gives the length of the fast axis. + changes = np.flatnonzero(~np.isclose(slow, slow[0])) + if changes.size == 0: + continue + n_fast = int(changes[0]) + if n_fast < 2 or qx.size % n_fast: + continue + shape = (qx.size // n_fast, n_fast) + fast_grid, slow_grid = fast.reshape(shape), slow.reshape(shape) + if np.allclose(fast_grid, fast_grid[0], equal_nan=True) and \ + np.allclose(slow_grid, slow_grid[:, :1], equal_nan=True): + return shape + return None + + +def reshape(values, shape: tuple[int, ...] | None): + """Reshape values to the grid shape, if there is one and the values fit it.""" + values = np.asarray(values) + if shape is not None and values.size == int(np.prod(shape)): + return values.reshape(shape) + return values From 8d89c6f543b79074a6c1898a3879c6d13f94ecbc Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 04:12:45 -0500 Subject: [PATCH 06/13] Create base cansas xml exporter --- sasdata/data_io/exporters/export_xml.py | 251 ++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 sasdata/data_io/exporters/export_xml.py diff --git a/sasdata/data_io/exporters/export_xml.py b/sasdata/data_io/exporters/export_xml.py new file mode 100644 index 000000000..f62933b4f --- /dev/null +++ b/sasdata/data_io/exporters/export_xml.py @@ -0,0 +1,251 @@ +""" +Export SasData to canSAS 1D XML files (version 1.1). + +The files are valid against the canSAS 1.1 schema (http://www.cansas.org/formats/1.1/cansas1d.xsd), so they can be +read by both :class:`sasdata.data_io.importers.import_xml.XmlImporter` and SasView. The format only holds 1D data. +""" + +import math +from pathlib import Path + +from lxml import etree + +from sasdata.data import SasMeasurement +from sasdata.data_io.exporter import Exporter, data_contents +from sasdata.dataset_types import one_dim +from sasdata.metadata import ( + Aperture, + BeamSize, + Collimation, + Detector, + Instrument, + Metadata, + Process, + Rot3, + Sample, + Source, + Vec3, +) +from sasdata.quantities.quantity import Quantity + +CANSAS_VERSION = "1.1" +CANSAS_NAMESPACE = "urn:cansas1d:1.1" +CANSAS_SCHEMA = "http://www.cansas.org/formats/1.1/cansas1d.xsd" +XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" + + +def tag(name: str) -> str: + """The fully qualified name of a canSAS element""" + return f"{{{CANSAS_NAMESPACE}}}{name}" + + +def format_float(value) -> str: + """Format a number using the XML schema float representation""" + value = float(value) + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "INF" if value > 0 else "-INF" + return repr(value) + + +def sub_element(parent: etree._Element, element_name: str, text: str | None = None, /, **attrs) -> etree._Element: + """Add a child element, with optional text and attributes (attributes with a value of None are skipped)""" + element = etree.SubElement(parent, tag(element_name), {k: str(v) for k, v in attrs.items() if v is not None}) + if text is not None: + element.text = text + return element + + +def quantity_element(parent: etree._Element, element_name: str, quantity: Quantity | None, /, **attrs): + """Add an element holding a single value with units, if the quantity is present""" + if quantity is None: + return None + value = quantity.value + if not isinstance(value, (int, float)): + value = value.flat[0] if hasattr(value, "flat") else value[0] + return sub_element(parent, element_name, format_float(value), unit=quantity.units.ascii_symbol, **attrs) + + +def vec3_element(parent: etree._Element, element_name: str, vector: Vec3 | None, name: str | None = None): + if vector is None: + return + element = sub_element(parent, element_name, name=name) + quantity_element(element, "x", vector.x) + quantity_element(element, "y", vector.y) + quantity_element(element, "z", vector.z) + + +def rot3_element(parent: etree._Element, element_name: str, rotation: Rot3 | None): + if rotation is None: + return + element = sub_element(parent, element_name) + quantity_element(element, "roll", rotation.roll) + quantity_element(element, "pitch", rotation.pitch) + quantity_element(element, "yaw", rotation.yaw) + + +class CanSASXmlExporter(Exporter): + """Exporter for canSAS 1D XML (v1.1) files. Supports 1D data only, with any number of datasets per file.""" + + name = "canSAS XML" + aliases = ("cansas", "xml", "canSAS 1D XML") + extensions = (".xml",) + supported_dataset_types = (one_dim,) + multiple_datasets = True + + def write(self, data: dict[str, SasMeasurement], path: Path): + root = self.build_tree(data) + etree.ElementTree(root).write(str(path), xml_declaration=True, encoding="UTF-8", pretty_print=True) + + def build_tree(self, data: dict[str, SasMeasurement]) -> etree._Element: + """Build the XML tree for a list of datasets""" + root = etree.Element(tag("SASroot"), nsmap={None: CANSAS_NAMESPACE, "xsi": XSI_NAMESPACE}) + root.set("version", CANSAS_VERSION) + root.set(f"{{{XSI_NAMESPACE}}}schemaLocation", f"{CANSAS_NAMESPACE} {CANSAS_SCHEMA}") + for name, datum in data.items(): + self.write_entry(root, name, datum) + return root + + def write_entry(self, root: etree._Element, name: str, datum: SasMeasurement): + """Write a single dataset to an SASentry, in the order required by the canSAS schema""" + metadata: Metadata = datum.metadata + entry = sub_element(root, "SASentry", name=name) + sub_element(entry, "Title", metadata.title if metadata.title is not None else "") + for run in metadata.run or [""]: + sub_element(entry, "Run", run) + self.write_data(entry, datum) + self.write_sample(entry, metadata.sample) + self.write_instrument(entry, metadata.instrument) + for process in metadata.process or []: + self.write_process(entry, process) + # At least one SASnote is required by the schema. + sub_element(entry, "SASnote") + + # ------------------------------------------------------------------------------------------------------------------ + # Data + # ------------------------------------------------------------------------------------------------------------------ + + @staticmethod + def write_data(entry: etree._Element, datum: SasMeasurement): + contents = data_contents(datum) + q = contents["Q"] + intensity = contents["I"] + length = len(q.value) + + # Each column is (element name, values, unit). They are written in the order required by the schema. + columns: list[tuple[str, object, str | None]] = [ + ("Q", q.value, q.units.ascii_symbol), + ("I", intensity.value, intensity.units.ascii_symbol), + ] + if intensity.has_error: + columns.append(("Idev", intensity.standard_error.value, intensity.units.ascii_symbol)) + # The schema allows either Qdev, or dQw and dQl. + if q.has_error: + columns.append(("Qdev", q.standard_error.value, q.units.ascii_symbol)) + elif "dQ" in contents: + columns.append(("Qdev", contents["dQ"].in_units_of(q.units), q.units.ascii_symbol)) + else: + for slit in ("dQw", "dQl"): + if slit in contents: + columns.append((slit, contents[slit].value, contents[slit].units.ascii_symbol)) + if "Qmean" in contents: + columns.append(("Qmean", contents["Qmean"].value, contents["Qmean"].units.ascii_symbol)) + if "Shadowfactor" in contents: + # Shadowfactor is dimensionless and has no unit attribute in the schema. + columns.append(("Shadowfactor", contents["Shadowfactor"].value, None)) + + sasdata = sub_element(entry, "SASdata") + for index in range(length): + idata = sub_element(sasdata, "Idata") + for column, values, unit in columns: + sub_element(idata, column, format_float(values[index]), unit=unit) + + # ------------------------------------------------------------------------------------------------------------------ + # Metadata + # ------------------------------------------------------------------------------------------------------------------ + + @staticmethod + def write_sample(entry: etree._Element, sample: Sample | None): + element = sub_element(entry, "SASsample", name=sample.name if sample else None) + # ID is required by the schema. + sub_element(element, "ID", sample.sample_id if sample and sample.sample_id is not None else "") + if sample is None: + return + quantity_element(element, "thickness", sample.thickness) + if sample.transmission is not None: + sub_element(element, "transmission", format_float(sample.transmission)) + quantity_element(element, "temperature", sample.temperature) + vec3_element(element, "position", sample.position) + rot3_element(element, "orientation", sample.orientation) + for detail in sample.details or []: + sub_element(element, "details", detail) + + def write_instrument(self, entry: etree._Element, instrument: Instrument | None): + element = sub_element(entry, "SASinstrument") + # The instrument name, source, a collimation and a detector are all required by the schema. + sub_element(element, "name", "") + self.write_source(element, instrument.source if instrument else None) + for collimation in (instrument.collimations if instrument else None) or [None]: + self.write_collimation(element, collimation) + for detector in (instrument.detector if instrument else None) or [None]: + self.write_detector(element, detector) + + @staticmethod + def write_source(parent: etree._Element, source: Source | None): + element = sub_element(parent, "SASsource") + sub_element(element, "radiation", source.radiation if source and source.radiation is not None else "") + if source is None: + return + beam_size: BeamSize | None = source.beam_size + if beam_size is not None: + vec3_element(element, "beam_size", beam_size.size or Vec3(x=None, y=None, z=None), beam_size.name) + if source.beam_shape is not None: + sub_element(element, "beam_shape", source.beam_shape) + quantity_element(element, "wavelength", source.wavelength) + quantity_element(element, "wavelength_min", source.wavelength_min) + quantity_element(element, "wavelength_max", source.wavelength_max) + quantity_element(element, "wavelength_spread", source.wavelength_spread) + + @staticmethod + def write_collimation(parent: etree._Element, collimation: Collimation | None): + element = sub_element(parent, "SAScollimation") + if collimation is None: + return + quantity_element(element, "length", collimation.length) + for aperture in collimation.apertures: + aperture: Aperture + aperture_element = sub_element(element, "aperture", name=aperture.name, type=aperture.type_) + vec3_element(aperture_element, "size", aperture.size, aperture.size_name) + quantity_element(aperture_element, "distance", aperture.distance) + + @staticmethod + def write_detector(parent: etree._Element, detector: Detector | None): + element = sub_element(parent, "SASdetector") + sub_element(element, "name", detector.name if detector and detector.name is not None else "") + if detector is None: + return + quantity_element(element, "SDD", detector.distance) + vec3_element(element, "offset", detector.offset) + rot3_element(element, "orientation", detector.orientation) + vec3_element(element, "beam_center", detector.beam_center) + vec3_element(element, "pixel_size", detector.pixel_size) + quantity_element(element, "slit_length", detector.slit_length) + + @staticmethod + def write_process(entry: etree._Element, process: Process): + element = sub_element(entry, "SASprocess") + if process.name is not None: + sub_element(element, "name", process.name) + if process.date is not None: + sub_element(element, "date", process.date) + if process.description is not None: + sub_element(element, "description", process.description) + for name, value in (process.terms or {}).items(): + if isinstance(value, Quantity): + quantity_element(element, "term", value, name=name) + else: + sub_element(element, "term", str(value), name=name) + # At least one SASprocessnote is required by the schema. + for note in process.notes or [""]: + sub_element(element, "SASprocessnote", note) From 0122ad5615eb859c24e84da1033bdb13d2c7bf9e Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 04:15:49 -0500 Subject: [PATCH 07/13] Remove data_contents calls from exporters that was removed from the main exporter to eliminate unwanted unit conversion --- sasdata/data_io/exporters/export_ascii.py | 2 +- sasdata/data_io/exporters/export_nxcansas.py | 4 ++-- sasdata/data_io/exporters/export_xml.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sasdata/data_io/exporters/export_ascii.py b/sasdata/data_io/exporters/export_ascii.py index 02d5f63c4..4a82adc78 100644 --- a/sasdata/data_io/exporters/export_ascii.py +++ b/sasdata/data_io/exporters/export_ascii.py @@ -83,7 +83,7 @@ def build_columns(self, datum: SasMeasurement) -> tuple[list[str], list[np.ndarr Find the columns to write. Returns the column headings (name and units) and the values of each column, in the units given in the headings. """ - contents = {name: quantity for name, quantity in datum.data_contents.items()} + contents = {name: quantity for name, quantity in datum._data_contents.items()} positional, extras = COLUMN_ORDERS[datum.dataset_type.name] available: dict[str, tuple[np.ndarray, Unit]] = {} for name, quantity in contents.items(): diff --git a/sasdata/data_io/exporters/export_nxcansas.py b/sasdata/data_io/exporters/export_nxcansas.py index 378ee7cc6..f3bd1b0c8 100644 --- a/sasdata/data_io/exporters/export_nxcansas.py +++ b/sasdata/data_io/exporters/export_nxcansas.py @@ -14,7 +14,7 @@ import numpy as np from sasdata.data import SasMeasurement -from sasdata.data_io.exporter import Exporter, data_contents +from sasdata.data_io.exporter import Exporter from sasdata.dataset_types import one_dim, three_dim, two_dim from sasdata.quantities.quantity import Quantity @@ -86,7 +86,7 @@ def write_entry(self, entry: h5py.Group, key: str, datum: SasMeasurement): @staticmethod def write_data(group: h5py.Group, datum: SasMeasurement): """Write the quantities of a dataset to a SASdata group.""" - contents = data_contents(datum) + contents = {name: quantity for name, quantity in datum._data_contents.items()} axes = AXES[datum.dataset_type.name] shape = grid_shape(contents, axes) diff --git a/sasdata/data_io/exporters/export_xml.py b/sasdata/data_io/exporters/export_xml.py index f62933b4f..c45469f6b 100644 --- a/sasdata/data_io/exporters/export_xml.py +++ b/sasdata/data_io/exporters/export_xml.py @@ -11,7 +11,7 @@ from lxml import etree from sasdata.data import SasMeasurement -from sasdata.data_io.exporter import Exporter, data_contents +from sasdata.data_io.exporter import Exporter from sasdata.dataset_types import one_dim from sasdata.metadata import ( Aperture, @@ -128,7 +128,7 @@ def write_entry(self, root: etree._Element, name: str, datum: SasMeasurement): @staticmethod def write_data(entry: etree._Element, datum: SasMeasurement): - contents = data_contents(datum) + contents = {name: quantity for name, quantity in datum._data_contents.items()} q = contents["Q"] intensity = contents["I"] length = len(q.value) From 58709fc00def24c2bf23447065827379f99aa737 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 04:16:29 -0500 Subject: [PATCH 08/13] Create __init__ for exporters package to allow a single point of import for all exporters --- sasdata/data_io/exporters/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sasdata/data_io/exporters/__init__.py b/sasdata/data_io/exporters/__init__.py index e69de29bb..1ca06ba4f 100644 --- a/sasdata/data_io/exporters/__init__.py +++ b/sasdata/data_io/exporters/__init__.py @@ -0,0 +1,14 @@ +""" +Exporters for each of the file formats supported by sasdata. + +Each module defines one or more children of :class:`sasdata.data_io.exporter.Exporter`. Importing this package registers +all of them, so :meth:`Exporter.export_data` can select the correct one by name or from a file extension. +""" + +from sasdata.data_io.exporters.export_ascii import AsciiExporter, CsvExporter +from sasdata.data_io.exporters.export_nxcansas import NXcanSASExporter +from sasdata.data_io.exporters.export_xml import CanSASXmlExporter + +__all__ = [ + 'AsciiExporter', 'CanSASXmlExporter', 'CsvExporter', 'NXcanSASExporter', +] From 3e05359ba93054a5ead5e85d47188874cc4d5f8f Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 05:05:38 -0500 Subject: [PATCH 09/13] Update exporter documentation explicitly stating no units are converted on export in order to retain the originally loaded units --- sasdata/data_io/exporter.py | 9 ++++++++- sasdata/data_io/exporters/export_ascii.py | 4 +++- sasdata/data_io/exporters/export_nxcansas.py | 3 +++ sasdata/data_io/exporters/export_xml.py | 5 +++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/sasdata/data_io/exporter.py b/sasdata/data_io/exporter.py index dee77685d..82eac400c 100644 --- a/sasdata/data_io/exporter.py +++ b/sasdata/data_io/exporter.py @@ -7,7 +7,14 @@ Children register themselves automatically when they are defined. :meth:`Exporter.export_data` selects the exporter for each file, either by name or from the file extension, checks that the destination can be written to, and then -writes the file:: +writes the file. + +Units are never converted: every quantity is written in the units it has in the data, and those units are recorded in +the file (in the column headings of text files and the unit attributes of HDF5 and XML files), so the sasdata importers +read the data back unchanged. SasView's own loaders assume 1/Å and 1/cm for text files, do not convert the units of +HDF5 files, and only convert the XML units that they know (which do not include nm^-1), so data should already be in +1/Å and 1/cm to be read correctly by SasView. The one exception is the canSAS XML format, which can only hold the Q +resolution in the units of Q, so a separate ``dQ`` quantity is converted to the units of Q:: from sasdata.data_io.exporter import Exporter diff --git a/sasdata/data_io/exporters/export_ascii.py b/sasdata/data_io/exporters/export_ascii.py index 4a82adc78..16da0982b 100644 --- a/sasdata/data_io/exporters/export_ascii.py +++ b/sasdata/data_io/exporters/export_ascii.py @@ -5,7 +5,9 @@ ``Q(Ang^-1) I(cm^-1) dI(cm^-1) dQ(Ang^-1)``, so :class:`sasdata.data_io.importers.import_ascii.AsciiImporter` reads the columns back with the correct names and units. -The columns are written in the order SasView expects, in the units SasView assumes (1/Å and 1/cm): +The columns are written in the order SasView expects, in the units of the data. Nothing is converted, so SasView, which +assumes 1/Å and 1/cm for text files and ignores the units in the headings, only reads data that is already in those +units correctly: * 1D data: ``Q I dI dQ``, then any of ``dQw dQl Qmean Shadowfactor``. SasView reads these from ``.txt`` and ``.csv`` files. diff --git a/sasdata/data_io/exporters/export_nxcansas.py b/sasdata/data_io/exporters/export_nxcansas.py index f3bd1b0c8..ed573b35c 100644 --- a/sasdata/data_io/exporters/export_nxcansas.py +++ b/sasdata/data_io/exporters/export_nxcansas.py @@ -5,6 +5,9 @@ written by SasView: each dataset is written to its own SASentry, with its data in a SASdata group. Uncertainties are linked to their data with the ``uncertainties`` attribute and resolutions to Q with the ``resolutions`` attribute, so the files can be read by both :class:`sasdata.data_io.importers.import_hdf5.HDF5Importer` and SasView. + +Quantities are written in their own units, which are not converted. SasView does not convert the units of HDF5 files +either, so data should be in 1/Å and 1/cm to be read correctly by SasView. """ import re diff --git a/sasdata/data_io/exporters/export_xml.py b/sasdata/data_io/exporters/export_xml.py index c45469f6b..45bb832fb 100644 --- a/sasdata/data_io/exporters/export_xml.py +++ b/sasdata/data_io/exporters/export_xml.py @@ -3,6 +3,11 @@ The files are valid against the canSAS 1.1 schema (http://www.cansas.org/formats/1.1/cansas1d.xsd), so they can be read by both :class:`sasdata.data_io.importers.import_xml.XmlImporter` and SasView. The format only holds 1D data. + +Quantities are written in their own units, which are not converted. The only exception is a separate ``dQ`` quantity, +which is converted to the units of Q because the format holds a single unit for Q and its resolution. SasView only +converts XML units that it knows (which do not include nm^-1), so data should be in 1/Å and 1/cm to be read correctly +by SasView. """ import math From 52a31d04d1cea472b6c4be186c71ab10a7074f76 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 05:06:50 -0500 Subject: [PATCH 10/13] Add unit parser to the ascii importer to better enable unit handling on import --- sasdata/data_io/importers/import_ascii.py | 96 +++++++++++++++++++++-- 1 file changed, 88 insertions(+), 8 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index c24bbb50a..a90050593 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -15,7 +15,7 @@ ) from sasdata.data import SasMeasurement from sasdata.data_io.importer import Importer, Source_Type, is_text, measurements_by_name, read_header -from sasdata.dataset_types import DatasetType, one_dim, unit_kinds +from sasdata.dataset_types import DatasetType, angle_dim, one_dim, sesans, three_dim, two_dim, unit_kinds from sasdata.default_units import get_default_unit from sasdata.guess import ( guess_column_count, @@ -24,11 +24,15 @@ guess_starting_position, ) from sasdata.metadata import Metadata, MetaNode +from sasdata.quantities import unit_parser, units from sasdata.quantities.quantity import NamedQuantity, Quantity from sasdata.quantities.units import NamedUnit logger = logging.getLogger(__name__) +# Dataset types that the columns in a header line are matched against, in order of preference. +all_dataset_types = [one_dim, two_dim, three_dim, angle_dim, sesans] + class AsciiSeparator(Enum): Comma = (0,) @@ -91,22 +95,98 @@ def guess_params_from_filename(filename: str, dataset_type: DatasetType) -> Asci lines_split = [split_line(separator_dict, line) for line in lines] startpos = guess_starting_position(lines_split) colcount = guess_column_count(lines_split, startpos) - # Columns we don't know the unit for are ignored. They are kept as rather than removed, so the - # columns after them still line up with the data. - columns = [ - (x, get_default_unit(x, unit_kinds[x])) if x in unit_kinds else ("", None) - for x in guess_columns(colcount, dataset_type) - ] + header = find_column_header(lines_split, startpos, colcount) + if header is not None: + # The file names its columns, so use those (and their units) instead of guessing. + columns, dataset_type = header + else: + # Columns we don't know the unit for are ignored. They are kept as rather than removed, so the + # columns after them still line up with the data. + columns = [ + (x, get_default_unit(x, unit_kinds[x])) if x in unit_kinds else ("", None) + for x in guess_columns(colcount, dataset_type) + ] + dataset_type = guess_dataset_type(filename) params = AsciiReaderParams( [filename], columns, starting_line=startpos, separator_dict=separator_dict, - dataset_type=guess_dataset_type(filename), + dataset_type=dataset_type, ) return params +# Every column name that can appear in a dataset, which may be used to name columns in a header line. +KNOWN_COLUMNS = set(unit_kinds) | { + column for dataset in all_dataset_types for column in dataset.required + dataset.optional +} + +# A column heading: a name optionally followed by units in brackets, e.g. 'Q', 'Q(Ang^-1)', 'I[cm^-1]'. +COLUMN_HEADING = re.compile(r"^(?P|[A-Za-z][A-Za-z0-9_]*)(?:\((?P[^()]*)\)|\[(?P[^\]]*)\])?$") + + +def parse_column_header(tokens: list[str]) -> tuple[list[tuple[str, NamedUnit | None]], DatasetType] | None: + """ + Try to read a line of tokens as column headings, each of which is the name of a quantity optionally followed by + its units, e.g. ``Q(Ang^-1) I(cm^-1) dI(cm^-1)``. Columns named ```` are skipped. + + :return: The columns and the dataset type they make up, or None if the tokens are not all recognised headings. + """ + tokens = [token for token in tokens if token] + if tokens and tokens[0].startswith("#"): + tokens[0] = tokens[0].lstrip("#") + tokens = [token for token in tokens if token] + columns: list[tuple[str, NamedUnit | None]] = [] + for token in tokens: + match = COLUMN_HEADING.match(token) + if match is None: + return None + name = match.group("name") + if name == "": + columns.append((name, None)) + continue + if name not in KNOWN_COLUMNS: + return None + unit_string = match.group("unit") or match.group("bracket_unit") + if unit_string: + try: + unit = unit_parser.parse(unit_string.strip()) + except Exception: + return None + elif name in unit_kinds: + unit = get_default_unit(name, unit_kinds[name]) + else: + unit = units.none + if not isinstance(unit, NamedUnit): + return None + columns.append((name, unit)) + + names = [name for name, _ in columns if name != ""] + if len(set(names)) != len(names): + return None + for dataset_type in all_dataset_types: + if all(name in names for name in dataset_type.required) and \ + all(name in dataset_type.required + dataset_type.optional for name in names): + return columns, replace(dataset_type) + return None + + +def find_column_header(lines_split: list[list[str]], startpos: int, colcount: int) \ + -> tuple[list[tuple[str, NamedUnit | None]], DatasetType] | None: + """Look back through the lines before the data for one that names each column.""" + for line in reversed(lines_split[:startpos]): + tokens = [token for token in line if token] + if tokens and tokens[0] == "#": + tokens = tokens[1:] + if len(tokens) != colcount: + continue + header = parse_column_header(tokens) + if header is not None: + return header + return None + + def split_line(separator_dict: dict[str, bool], line: str) -> list[str]: """Split a line in a CSV file based on which seperators the user has selected on the widget. From 44511cd795d210a043581373330240434770bfd2 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 05:07:53 -0500 Subject: [PATCH 11/13] update the hfd5 resolution importing to ensure dQ values are treated as resolutions and not as uncertainties --- sasdata/data_io/importers/import_hdf5.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sasdata/data_io/importers/import_hdf5.py b/sasdata/data_io/importers/import_hdf5.py index 4968b4721..3902b49fd 100644 --- a/sasdata/data_io/importers/import_hdf5.py +++ b/sasdata/data_io/importers/import_hdf5.py @@ -107,8 +107,11 @@ def connected_data(node: SASDataGroup, name_prefix="", metadata=None) -> dict[st uncertainty_map[name] = uncertainty_name uncertainties.add(uncertainty_name) if "resolutions" in child.attributes: - resolutions_name = child.attributes["resolutions"] - resolutions.add(resolutions_name) + # A single name, or an array of names when there are several resolutions (e.g. slit smearing) + for resolutions_name in np.atleast_1d(child.attributes["resolutions"]): + if isinstance(resolutions_name, bytes): + resolutions_name = resolutions_name.decode() + resolutions.add(str(resolutions_name)) entries[name] = quantity From 524f2e74e14bd6604ef9a531e14c2f63fd53c785 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 05:08:48 -0500 Subject: [PATCH 12/13] add ascii importer tests to test header parsing for units --- test/data_io/utest_import_ascii.py | 39 ++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/test/data_io/utest_import_ascii.py b/test/data_io/utest_import_ascii.py index 1fa989443..be88bed60 100644 --- a/test/data_io/utest_import_ascii.py +++ b/test/data_io/utest_import_ascii.py @@ -10,7 +10,12 @@ ) from sasdata.dataset_types import one_dim from sasdata.guess import guess_columns -from sasdata.quantities.units import per_angstrom, per_centimeter +from sasdata.quantities.units import ( + per_angstrom, + per_centimeter, + per_meter, + per_nanometer, +) from test import local_data_finder, local_mumag_finder # TODO: These are using the private _data_contents temporarily. Later, there will be a public way of accessing these, @@ -168,7 +173,7 @@ def load_text(tmp_path, text: str, columns=Q_I_DI_COLUMNS, **kwargs) -> dict: separator_dict=ALL_SEPARATORS, **kwargs, ) - return load_data(params)[0]._data_contents + return list(AsciiImporter(params=params).load_data().values())[0]._data_contents # Line numbers in the comments are the (0-based) indices used for excluded_lines. @@ -271,3 +276,33 @@ def test_column_without_unit_is_ignored(tmp_path): assert "dQ" not in data assert data["Q"].value == pytest.approx([0.01, 0.02]) assert data["I"].value == pytest.approx([100, 90]) + + +def test_column_headings(tmp_path): + """Column names and units in a heading line are used instead of guessing them""" + filename = tmp_path / "headings.txt" + filename.write_text( + "# A comment\n" + "Q(nm^-1) I(m^-1) dI(m^-1)\n" + "0.1 99 10 1\n" + "0.2 99 20 2\n" + "0.3 99 30 3\n" + ) + params = guess_params_from_filename(str(filename), one_dim) + assert [name for name, _ in params.columns] == ["Q", "", "I", "dI"] + data = list(AsciiImporter().load_data(filename).values())[0] + assert data.dataset_type.name == one_dim.name + # The units in the headings are kept as they are, and not converted to 1/Å and 1/cm + assert data["Q"].units == per_nanometer + assert data["I"].units == per_meter + assert data["I"].value == pytest.approx([10, 20, 30]) + assert data["I"].standard_error.value == pytest.approx([1, 2, 3]) + assert data["Q"].value == pytest.approx([0.1, 0.2, 0.3]) + + +def test_unrecognised_headings_are_guessed(tmp_path): + """Headings that are not all known column names are ignored, and the columns guessed as before""" + filename = tmp_path / "unknown.txt" + filename.write_text(" \n0.1 10 1\n0.2 20 2\n0.3 30 3\n") + params = guess_params_from_filename(str(filename), one_dim) + assert [name for name, _ in params.columns] == ["Q", "I", "dI"] From 083efb498d55c6ef75468c50a4c33a3ab1866c31 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 9 Nov 2026 05:09:40 -0500 Subject: [PATCH 13/13] add a number of exporter tests, including testing the new exporters against legacy loaders as well as the new importers --- test/data_io/utest_exporters.py | 526 ++++++++++++++++++++++++++++++++ 1 file changed, 526 insertions(+) create mode 100644 test/data_io/utest_exporters.py diff --git a/test/data_io/utest_exporters.py b/test/data_io/utest_exporters.py new file mode 100644 index 000000000..d6fe4290e --- /dev/null +++ b/test/data_io/utest_exporters.py @@ -0,0 +1,526 @@ +""" +Unit tests for the Exporter class and the NXcanSAS, canSAS XML, ASCII and CSV exporters. + +Every exporter is tested by exporting data loaded from the test files and importing it again, with both the sasdata +importers and the data loader used by SasView, to check that exported data can be read back in. +""" + +import os +import stat +from pathlib import Path + +import h5py +import numpy as np +import pytest +from lxml import etree + +from sasdata.data import SasMeasurement +from sasdata.data_io.exporter import Exporter, ExportError, default_file_mode +from sasdata.data_io.exporters import AsciiExporter, CanSASXmlExporter, CsvExporter, NXcanSASExporter +from sasdata.data_io.importer import Importer +from sasdata.dataloader.data_info import Data1D, Data2D +from sasdata.dataloader.loader import Loader +from sasdata.dataset_types import one_dim, sesans, two_dim +from sasdata.quantities import units +from sasdata.quantities.quantity import NamedQuantity +from sasdata.quantities.units import Unit +from test import local_data_finder, local_sesans_finder + +SCHEMA = Path(__file__).parents[2] / "sasdata" / "dataloader" / "readers" / "schema" / "cansas1d_v1_1.xsd" + +# Source files for the round trip tests, and the formats each one can be exported to. +ONE_DIM_FILES = ["ISIS_1_1.xml", "cansas1d.xml", "cansas1d_slit.xml", "simpleexamplefile.h5", "test_3_columns.txt"] +TWO_DIM_FILES = ["MAR07232_rest.h5"] +MIXED_FILES = ["nxcansas_1Dand2D_multisasentry.h5"] + +# The format each exporter writes, as identified by file_format +EXPECTED_FORMAT = { + NXcanSASExporter: "hdf5", + CanSASXmlExporter: "xml", + AsciiExporter: "ascii", + CsvExporter: "ascii", +} + + +def file_format(path: str | Path) -> str: + """Identify the format of a file from its contents, so the extension of an exported file is not relied on""" + if h5py.is_hdf5(path): + return "hdf5" + with open(path, "rb") as f: + if f.read(1024).lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"<"): + return "xml" + return "ascii" + + +def import_file(path: str | Path) -> dict[str, SasMeasurement]: + """Import a file, letting the Importer infer the format from the file contents rather than its extension""" + return Importer.load(path) + + +def load(filename: str) -> dict[str, SasMeasurement]: + return import_file(local_data_finder(filename)) + + +def first(data: dict[str, SasMeasurement]) -> SasMeasurement: + return next(iter(data.values())) + + +def make_1d(name: str = "synthetic", points: int = 20, slit: bool = False, + q_units: Unit = units.per_nanometer) -> SasMeasurement: + """ + Synthetic 1D data, with Q in nm^-1 by default so the exporters are tested with units other than those SasView + assumes. Use ``q_units=units.per_angstrom`` for data that SasView's loaders can read correctly. + """ + q = np.linspace(0.001, 0.2, points) + q_scale = 1.0 if q_units == units.per_angstrom else 10.0 + contents = { + "Q": NamedQuantity("Q", q * q_scale, q_units, standard_error=q * 0.5), + "I": NamedQuantity("I", 1 / q, units.per_centimeter, standard_error=0.1 / q), + } + if slit: + contents["Q"] = NamedQuantity("Q", q, units.per_angstrom) + contents["dQw"] = NamedQuantity("dQw", np.full(points, 0.001), units.per_angstrom) + contents["dQl"] = NamedQuantity("dQl", np.full(points, 0.1), units.per_angstrom) + return SasMeasurement(name, contents, one_dim) + + +def make_2d(name: str = "synthetic 2D", nx: int = 12, ny: int = 10) -> SasMeasurement: + qx, qy = np.meshgrid(np.linspace(-0.1, 0.1, nx), np.linspace(-0.08, 0.08, ny)) + intensity = np.exp(-(qx ** 2 + qy ** 2) * 100) + contents = { + "Qx": NamedQuantity("Qx", qx.reshape(-1), units.per_angstrom), + "Qy": NamedQuantity("Qy", qy.reshape(-1), units.per_angstrom), + "I": NamedQuantity("I", intensity.reshape(-1), units.per_centimeter, standard_error=intensity.reshape(-1) / 10), + } + return SasMeasurement(name, contents, two_dim) + + +def assert_same_data(original: SasMeasurement, imported: SasMeasurement): + """ + The quantities (and their uncertainties) must match, allowing for changes of array shape. The exporters keep the + units of the data as they are, so the units must not change either. + """ + assert imported.dataset_type.name == original.dataset_type.name + imported_contents = imported._data_contents + for name, quantity in original._data_contents.items(): + if name == "dQ" and name not in imported_contents: + # Formats which only hold one Q resolution store dQ as the uncertainty in Q + assert np.allclose(np.ravel(quantity.in_units_of(imported_contents["Q"].units)), + np.ravel(imported_contents["Q"].standard_error.value), equal_nan=True) + continue + assert name in imported_contents, f"{name} was not imported" + result = imported_contents[name] + assert result.units == quantity.units, f"{name} changed from {quantity.units} to {result.units}" + assert np.allclose(np.ravel(quantity.in_units_of(result.units)), np.ravel(result.value), equal_nan=True) + if quantity.has_error: + assert result.has_error, f"The uncertainty in {name} was not imported" + assert np.allclose(np.ravel(quantity.standard_error.in_units_of(result.units)), + np.ravel(result.standard_error.value), equal_nan=True) + + +def legacy_load(path: Path) -> list[Data1D | Data2D]: + """Load a file with the data loader used by SasView""" + loaded = Loader().load(str(path)) + for datum in loaded: + assert not datum.errors, f"SasView reported errors loading {path}: {datum.errors}" + return loaded + + +def assert_legacy_matches(original: SasMeasurement, legacy: Data1D | Data2D): + """ + The data read by SasView must match the original data. + + SasView assumes 1/Å and 1/cm for text files and does not convert the units of HDF5 files, and the exporters keep the + units of the data as they are, so the original data must already be in SasView's units to be read back correctly. + """ + contents = original._data_contents + for name, unit in {"Q": units.per_angstrom, "Qx": units.per_angstrom, "Qy": units.per_angstrom, + "I": units.per_centimeter}.items(): + if name in contents: + assert contents[name].units == unit, f"{name} is not in SasView's units, so SasView cannot read it back" + if original.dataset_type == one_dim: + assert isinstance(legacy, Data1D) + q = contents["Q"].in_units_of(units.per_angstrom) + # Some of SasView's readers drop points at Q = 0, and all of them sort the data by Q + kept = np.arange(q.size) if legacy.x.size == q.size else np.flatnonzero(q != 0) + order = kept[np.argsort(q[kept], kind="stable")] + legacy_order = np.argsort(legacy.x, kind="stable") + assert np.allclose(legacy.x[legacy_order], q[order]) + assert np.allclose(legacy.y[legacy_order], contents["I"].in_units_of(units.per_centimeter)[order], + equal_nan=True) + if contents["I"].has_error: + assert np.allclose(legacy.dy[legacy_order], + contents["I"].standard_error.in_units_of(units.per_centimeter)[order], equal_nan=True) + else: + assert isinstance(legacy, Data2D) + assert np.allclose(np.ravel(legacy.qx_data), np.ravel(contents["Qx"].in_units_of(units.per_angstrom))) + assert np.allclose(np.ravel(legacy.qy_data), np.ravel(contents["Qy"].in_units_of(units.per_angstrom))) + assert np.allclose(np.ravel(legacy.data), np.ravel(contents["I"].in_units_of(units.per_centimeter)), + equal_nan=True) + + +# ---------------------------------------------------------------------------------------------------------------------- +# Exporter selection +# ---------------------------------------------------------------------------------------------------------------------- + +class TestExporterSelection: + def test_registry(self): + exporters = Exporter.exporters() + for exporter in (NXcanSASExporter, CanSASXmlExporter, AsciiExporter, CsvExporter): + assert exporter in exporters + assert issubclass(exporter, Exporter) + assert len(exporters) == 4 + + @pytest.mark.parametrize("filename, expected", [ + ("data.h5", NXcanSASExporter), + ("data.H5", NXcanSASExporter), + ("data.nxs", NXcanSASExporter), + ("data.hdf5", NXcanSASExporter), + ("data.xml", CanSASXmlExporter), + ("data.txt", AsciiExporter), + ("data.dat", AsciiExporter), + ("data.csv", CsvExporter), + ("data", None), + ("data.unknown", None), + ]) + def test_find_exporter_by_extension(self, filename, expected): + assert Exporter.find_exporter(path=filename) is expected + + @pytest.mark.parametrize("name, expected", [ + ("NXcanSAS", NXcanSASExporter), + ("nxcansas", NXcanSASExporter), + ("HDF5", NXcanSASExporter), + ("canSAS XML", CanSASXmlExporter), + ("xml", CanSASXmlExporter), + ("ASCII", AsciiExporter), + ("csv", CsvExporter), + (CsvExporter, CsvExporter), + ("not a format", None), + ]) + def test_find_exporter_by_name(self, name, expected): + assert Exporter.find_exporter(name=name) is expected + + def test_name_overrides_extension(self, tmp_path): + """An exporter chosen by name is used whatever the file extension, and the importer finds it by content""" + data = load("ISIS_1_1.xml") + destination = tmp_path / "exported.dat" + assert Exporter().export_data({destination: data}, exporter="canSAS XML") == [] + assert file_format(destination) == "xml" + assert_same_data(first(data), first(import_file(destination))) + + def test_child_exporter_ignores_extension(self, tmp_path): + destination = tmp_path / "no_extension" + assert NXcanSASExporter().export_data({destination: make_1d()}) == [] + assert file_format(destination) == "hdf5" + + def test_unknown_extension(self, tmp_path): + errors = Exporter().export_data({tmp_path / "data.unknown": make_1d()}) + assert len(errors) == 1 + assert "Unable to determine the export format" in errors[0] + assert not (tmp_path / "data.unknown").exists() + + def test_unknown_exporter(self, tmp_path): + errors = Exporter().export_data({tmp_path / "data.h5": make_1d()}, exporter="not a format") + assert len(errors) == 1 + assert "Unknown exporter" in errors[0] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Destination validation +# ---------------------------------------------------------------------------------------------------------------------- + +class TestDestinationValidation: + def test_valid_destination(self, tmp_path): + assert Exporter.validate_destination(tmp_path / "data.h5") == (tmp_path / "data.h5").absolute() + # The check must not leave anything behind + assert list(tmp_path.iterdir()) == [] + + def test_missing_directory(self, tmp_path): + with pytest.raises(FileNotFoundError): + Exporter.validate_destination(tmp_path / "missing" / "data.h5") + errors = Exporter().export_data({tmp_path / "missing" / "data.h5": make_1d()}) + assert len(errors) == 1 and "does not exist" in errors[0] + + def test_parent_is_a_file(self, tmp_path): + (tmp_path / "file").write_text("") + with pytest.raises(NotADirectoryError): + Exporter.validate_destination(tmp_path / "file" / "data.h5") + + def test_destination_is_a_directory(self, tmp_path): + (tmp_path / "data.h5").mkdir() + with pytest.raises(IsADirectoryError): + Exporter.validate_destination(tmp_path / "data.h5") + + def test_no_overwrite(self, tmp_path): + destination = tmp_path / "data.csv" + destination.write_text("original") + with pytest.raises(FileExistsError): + Exporter.validate_destination(destination, overwrite=False) + errors = Exporter().export_data({destination: make_1d()}, overwrite=False) + assert len(errors) == 1 + assert destination.read_text() == "original" + + def test_overwrite(self, tmp_path): + destination = tmp_path / "data.csv" + destination.write_text("original") + assert Exporter().export_data({destination: make_1d()}) == [] + assert_same_data(make_1d(), first(import_file(destination))) + + def test_directory_not_writable(self, tmp_path, monkeypatch): + """os.access is patched so the test also works when run with administrator rights""" + real_access = os.access + monkeypatch.setattr(os, "access", lambda p, mode: False if Path(p) == tmp_path else real_access(p, mode)) + with pytest.raises(PermissionError): + Exporter.validate_destination(tmp_path / "data.h5") + errors = Exporter().export_data({tmp_path / "data.h5": make_1d()}) + assert len(errors) == 1 and "permission" in errors[0] + assert not (tmp_path / "data.h5").exists() + + def test_file_not_writable(self, tmp_path, monkeypatch): + destination = tmp_path / "data.h5" + destination.write_text("original") + real_access = os.access + monkeypatch.setattr(os, "access", lambda p, mode: False if Path(p) == destination else real_access(p, mode)) + with pytest.raises(PermissionError): + Exporter.validate_destination(destination) + + def test_directory_cannot_be_written(self, tmp_path, monkeypatch): + """Permission systems that os.access does not understand are caught by trying to create a file""" + def refuse(*args, **kwargs): + raise PermissionError("Access is denied") + monkeypatch.setattr("tempfile.mkstemp", refuse) + with pytest.raises(PermissionError): + Exporter.validate_destination(tmp_path / "data.h5") + + def test_failed_write_keeps_existing_file(self, tmp_path, monkeypatch): + destination = tmp_path / "data.xml" + destination.write_text("original") + + def fail(self, data, path): + Path(path).write_text("partial") + raise RuntimeError("Write failed") + monkeypatch.setattr(CanSASXmlExporter, "write", fail) + errors = Exporter().export_data({destination: make_1d()}) + assert len(errors) == 1 and "Write failed" in errors[0] + assert destination.read_text() == "original" + assert [p.name for p in tmp_path.iterdir()] == ["data.xml"] + + @pytest.mark.skipif(os.name == "nt", reason="POSIX file permissions") + def test_exported_file_permissions(self, tmp_path): + destination = tmp_path / "data.h5" + Exporter.export(make_1d(), destination) + # The export is written via a private temporary file, but must end up with normal file permissions + assert stat.S_IMODE(destination.stat().st_mode) == default_file_mode() + + +# ---------------------------------------------------------------------------------------------------------------------- +# Data validation +# ---------------------------------------------------------------------------------------------------------------------- + +class TestDataValidation: + @pytest.mark.parametrize("exporter, extension", [(CanSASXmlExporter, "xml"), (CsvExporter, "csv")]) + def test_one_dim_only(self, tmp_path, exporter, extension): + errors = Exporter().export_data({tmp_path / f"data.{extension}": make_2d()}) + assert len(errors) == 1 and "only supports" in errors[0] + with pytest.raises(ExportError): + exporter.export(make_2d(), tmp_path / f"data.{extension}") + + @pytest.mark.parametrize("extension", ["h5", "xml", "txt", "csv"]) + def test_sesans_not_supported(self, tmp_path, extension): + data = first(Importer.load(local_sesans_finder("sphere2micron.ses"))) + assert data.dataset_type == sesans + errors = Exporter().export_data({tmp_path / f"data.{extension}": data}) + assert len(errors) == 1 and "only supports" in errors[0] + + @pytest.mark.parametrize("extension", ["txt", "csv"]) + def test_one_dataset_per_text_file(self, tmp_path, extension): + errors = Exporter().export_data({tmp_path / f"data.{extension}": [make_1d("a"), make_1d("b")]}) + assert len(errors) == 1 and "one dataset per file" in errors[0] + + def test_no_data(self, tmp_path): + errors = Exporter().export_data({tmp_path / "data.h5": []}) + assert len(errors) == 1 and "no data" in errors[0] + + def test_errors_for_each_file(self, tmp_path): + """A failure exporting one file does not stop the others being exported""" + errors = Exporter().export_data({ + tmp_path / "good.h5": make_1d(), + tmp_path / "bad.xml": make_2d(), + tmp_path / "missing" / "bad.csv": make_1d(), + tmp_path / "good.txt": make_1d(), + }) + assert len(errors) == 2 + assert sorted(p.name for p in tmp_path.iterdir()) == ["good.h5", "good.txt"] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Round trips: export, then import with sasdata and with SasView's loader +# ---------------------------------------------------------------------------------------------------------------------- + +round_trip_cases = [ + (filename, exporter, extension) + for filename in ONE_DIM_FILES + for exporter, extension in [(NXcanSASExporter, "h5"), (CanSASXmlExporter, "xml"), (AsciiExporter, "txt"), + (CsvExporter, "csv")] +] + [ + (filename, exporter, extension) + for filename in TWO_DIM_FILES + for exporter, extension in [(NXcanSASExporter, "h5"), (AsciiExporter, "dat")] +] + [ + (filename, NXcanSASExporter, "h5") for filename in MIXED_FILES +] + + +@pytest.mark.parametrize("filename, exporter, extension", round_trip_cases) +def test_round_trip(tmp_path, filename, exporter, extension): + data = load(filename) + if not exporter.multiple_datasets: + data = dict(list(data.items())[:1]) + destination = tmp_path / f"exported.{extension}" + assert Exporter().export_data({destination: data}) == [] + + assert file_format(destination) == EXPECTED_FORMAT[exporter] + imported = import_file(destination) + assert len(imported) == len(data) + if exporter.multiple_datasets: + # The keys (and so the names used by SasView) are kept + assert list(imported.keys()) == list(data.keys()) + for original, result in zip(data.values(), imported.values()): + assert_same_data(original, result) + + +@pytest.mark.parametrize("filename, exporter, extension", round_trip_cases) +def test_sasview_can_load(tmp_path, filename, exporter, extension): + data = load(filename) + if not exporter.multiple_datasets: + data = dict(list(data.items())[:1]) + if any(len(np.ravel(d["I"].value)) < 5 for d in data.values()): + pytest.skip("SasView will not load data with fewer than 5 points") + for d in data.values(): + for name, unit in {"Q": units.per_angstrom, "Qx": units.per_angstrom, "Qy": units.per_angstrom, + "I": units.per_centimeter}.items(): + if name in d._data_contents and d[name].units != unit: + pytest.skip(f"{name} is in {d[name].units}, and SasView only reads data in 1/Å and 1/cm correctly") + destination = tmp_path / f"exported.{extension}" + assert Exporter().export_data({destination: data}) == [] + loaded = legacy_load(destination) + assert len(loaded) == len(data) + for original, legacy in zip(data.values(), loaded): + assert_legacy_matches(original, legacy) + + +@pytest.mark.parametrize("extension", ["h5", "xml", "txt", "csv"]) +def test_synthetic_1d(tmp_path, extension): + """Data in SasView's units is read correctly by SasView, and Q resolutions are kept""" + original = make_1d(q_units=units.per_angstrom) + destination = tmp_path / f"synthetic.{extension}" + Exporter.export(original, destination) + assert_same_data(original, first(import_file(destination))) + legacy = legacy_load(destination)[0] + assert_legacy_matches(original, legacy) + assert np.allclose(legacy.dx, original["Q"].standard_error.value) + + +@pytest.mark.parametrize("extension", ["h5", "xml", "txt", "csv"]) +def test_native_units_are_kept(tmp_path, extension): + """Data is exported in its own units, without conversion to the units SasView assumes (1/Å and 1/cm)""" + original = make_1d() + assert original["Q"].units == units.per_nanometer + destination = tmp_path / f"native.{extension}" + Exporter.export(original, destination) + result = first(import_file(destination)) + assert result["Q"].units == units.per_nanometer + assert result["I"].units == units.per_centimeter + # The numbers must be exactly as they were: not rescaled to 1/Å + assert np.allclose(result["Q"].value, original["Q"].value) + assert np.allclose(result["Q"].standard_error.value, original["Q"].standard_error.value) + assert np.allclose(result["I"].value, original["I"].value) + assert np.allclose(result["I"].standard_error.value, original["I"].standard_error.value) + # SasView is deliberately not used to read these files: it only converts units it knows, and does not know nm^-1 + assert_same_data(original, result) + + +@pytest.mark.parametrize("extension", ["h5", "xml"]) +def test_slit_resolution(tmp_path, extension): + original = make_1d(slit=True) + destination = tmp_path / f"slit.{extension}" + Exporter.export(original, destination) + assert_same_data(original, first(import_file(destination))) + legacy = legacy_load(destination)[0] + assert np.allclose(legacy.dxw, original["dQw"].value) + assert np.allclose(legacy.dxl, original["dQl"].value) + + +@pytest.mark.parametrize("extension", ["h5", "dat"]) +def test_synthetic_2d(tmp_path, extension): + """Flat lists of 2D points on a grid are written so SasView recognises them as 2D data""" + original = make_2d() + destination = tmp_path / f"synthetic.{extension}" + Exporter.export(original, destination) + assert_same_data(original, first(import_file(destination))) + legacy = legacy_load(destination)[0] + assert_legacy_matches(original, legacy) + assert np.allclose(legacy.err_data, original["I"].standard_error.value) + + +@pytest.mark.parametrize("extension", ["h5", "xml"]) +def test_multiple_datasets(tmp_path, extension): + data = [make_1d(name, q_units=units.per_angstrom) for name in ["first", "second", "first"]] + destination = tmp_path / f"multiple.{extension}" + Exporter.export(data, destination) + imported = import_file(destination) + assert list(imported.keys()) == ["first", "second", "first (2)"] + for original, result in zip(data, imported.values()): + assert_same_data(original, result) + assert len(legacy_load(destination)) == 3 + + +@pytest.mark.parametrize("filename", ["ISIS_1_1.xml", "cansas1d.xml", "cansas1d_units.xml", "valid_cansas_xml.xml"]) +@pytest.mark.parametrize("extension", ["h5", "xml"]) +def test_metadata_round_trip(tmp_path, filename, extension): + data = load(filename) + destination = tmp_path / f"metadata.{extension}" + Exporter.export(data, destination) + for original, result in zip(data.values(), import_file(destination).values()): + assert result.metadata.title == original.metadata.title + assert result.metadata.run == original.metadata.run + assert result.metadata.sample == original.metadata.sample + assert result.metadata.instrument == original.metadata.instrument + assert result.metadata.process == original.metadata.process + # NXcanSAS files must declare their definition + assert result.metadata.definition in (original.metadata.definition, "NXcanSAS") + + +@pytest.mark.parametrize("filename", ONE_DIM_FILES) +def test_xml_is_valid_cansas(tmp_path, filename): + """Exported XML must be valid against the canSAS 1.1 schema, even when metadata is missing""" + destination = tmp_path / "exported.xml" + Exporter.export(load(filename), destination) + schema = etree.XMLSchema(etree.parse(str(SCHEMA))) + document = etree.parse(str(destination)) + assert schema.validate(document), schema.error_log + + +def test_ascii_headings(tmp_path): + """The headings give the units of the data as they are, not those SasView assumes""" + destination = tmp_path / "headings.txt" + Exporter.export(make_1d(), destination) + lines = destination.read_text().splitlines() + data_start = next(i for i, line in enumerate(lines) if not line.startswith("#")) + assert lines[data_start] == "Q(nm^-1) I(cm^-1) dI(cm^-1) dQ(nm^-1)" + assert len(lines[data_start + 1].split()) == 4 + + +def test_ascii_headings_in_sasview_units(tmp_path): + destination = tmp_path / "headings.txt" + Exporter.export(make_1d(q_units=units.per_angstrom), destination) + lines = destination.read_text().splitlines() + assert "Q(Ang^-1) I(cm^-1) dI(cm^-1) dQ(Ang^-1)" in lines + + +def test_csv_headings(tmp_path): + destination = tmp_path / "headings.csv" + Exporter.export(make_1d(), destination) + lines = destination.read_text().splitlines() + assert "Q(nm^-1),I(cm^-1),dI(cm^-1),dQ(nm^-1)" in lines + assert len(lines[-1].split(",")) == 4