diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index 5bb90408..35fd533a 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -101,7 +101,7 @@ def guess_params_from_filename(filename: str, dataset_type: DatasetType) -> Asci columns, starting_line=startpos, separator_dict=separator_dict, - dataset_type=guess_dataset_type(filename), + dataset_type=dataset_type, ) return params diff --git a/sasdata/guess.py b/sasdata/guess.py index 9bc0a7dd..abe765ee 100644 --- a/sasdata/guess.py +++ b/sasdata/guess.py @@ -1,11 +1,24 @@ +import re + +import numpy as np from scipy.stats import mode from sasdata.dataset_types import DatasetType, one_dim, two_dim +def _is_blank(line: list[str]) -> bool: + return len(line) == 0 or line == [""] + + def guess_column_count(split_csv: list[list[str]], starting_pos: int) -> int: - """Guess the amount of columns present in the data.""" - candidate_lines = split_csv[starting_pos::] + """Guess the amount of columns present in the data. + + Blank lines are not counted, so files with an empty line between data rows + are not guessed as having a single column. + """ + candidate_lines = [line for line in split_csv[starting_pos::] if not _is_blank(line)] + if not candidate_lines: + return 0 return int(mode([len(line) for line in candidate_lines]).mode) @@ -34,29 +47,69 @@ def guess_columns(col_count: int, dataset_type: DatasetType) -> list[str]: symbols_to_ignore = [".", "-", "+", "e", "E"] +def _is_numeric_row(row: list[str]) -> bool: + for column in row: + amended_column = column + for symbol in symbols_to_ignore: + amended_column = amended_column.replace(symbol, "") + if not amended_column.isdigit(): + return False + return True + + def guess_starting_position(split_csv: list[list[str]]) -> int: - """Try to look for a line where the first item in the row can be converted - to a number. If such a line doesn't exist, try to look for a line where the - first item in the row can be converted to a number. If such a line doesn't - exist, then just return 0 as the starting position. + """Find the first line of the data block. + + The data block is made of numeric lines with the most common column count. + Numeric header lines with a different column count (for example the + parameter lines of an Anton Paar SAXSess file) are skipped. If no line is + numeric, then just return 0 as the starting position. """ - for i, row in enumerate(split_csv): - all_nums = True - for column in row: - amended_column = column - for symbol in symbols_to_ignore: - amended_column = amended_column.replace(symbol, "") - if not amended_column.isdigit(): - all_nums = False - break - if all_nums: - return i - return 0 - - -def guess_dataset_type(filename: str) -> DatasetType: - if filename.lower().endswith(".dat"): + numeric_rows = [(i, len(row)) for i, row in enumerate(split_csv) if not _is_blank(row) and _is_numeric_row(row)] + if not numeric_rows: + return 0 + data_column_count = int(mode([count for _, count in numeric_rows]).mode) + return next(i for i, count in numeric_rows if count == data_column_count) + + +def _split_all_separators(line: str) -> list[str]: + # Same splitting as the ASCII importer with every separator enabled. + return re.split(r"\s+|,|\t", line.strip()) + + +def guess_dataset_type(filename: str, split_csv: list[list[str]] | None = None) -> DatasetType: + """Guess whether an ASCII file holds 1D or 2D data. + + Only ``.dat`` files can be 2D. They are guessed from their contents: 2D data + has at least three columns, and its first column (Qx) repeats or wraps + around, while the Q column of 1D data is strictly monotonic. + + :param filename: The file to guess for. + :param split_csv: The file's lines, already split. Read from ``filename`` if not given. + """ + if not filename.lower().endswith(".dat"): + return one_dim + if split_csv is None: + with open(filename) as file: + split_csv = [_split_all_separators(line) for line in file.readlines()] + start = guess_starting_position(split_csv) + column_count = guess_column_count(split_csv, start) + if column_count < 3: + return one_dim + first_column = [] + for row in split_csv[start:]: + if len(row) < column_count: + continue + try: + first_column.append(float(row[0])) + except ValueError: + continue + first_column = np.array(first_column) + first_column = first_column[np.isfinite(first_column)] + if len(first_column) < 2: return two_dim - else: + steps = np.diff(first_column) + if np.all(steps > 0) or np.all(steps < 0): return one_dim + return two_dim diff --git a/test/data_io/utest_import_ascii.py b/test/data_io/utest_import_ascii.py index 46b707cc..e5960586 100644 --- a/test/data_io/utest_import_ascii.py +++ b/test/data_io/utest_import_ascii.py @@ -9,8 +9,8 @@ load_data, load_data_default_params, ) -from sasdata.dataset_types import one_dim -from sasdata.guess import guess_columns +from sasdata.dataset_types import one_dim, two_dim +from sasdata.guess import guess_column_count, guess_columns, guess_dataset_type from sasdata.quantities.units import per_angstrom, per_centimeter from test import local_data_finder, local_mumag_finder @@ -251,6 +251,34 @@ def test_extra_columns_are_ignored(tmp_path): assert data["I"].value == pytest.approx([100, 90]) +THREE_COLUMN_DATA = "0.01 100 1\n0.02 90 2\n0.03 80 3\n" + + +def test_guess_params_honours_requested_dataset_type(tmp_path): + # guess_dataset_type() classifies every .dat file as 2D. An explicit request + # for 1D must win, otherwise the 1D columns are checked against the 2D type. + filename = tmp_path / "data.dat" + filename.write_text(THREE_COLUMN_DATA) + params = guess_params_from_filename(str(filename), one_dim) + assert params.dataset_type is one_dim + data = load_data(params)[0] + assert data.dataset_type is one_dim + assert data["Q"].value == pytest.approx([0.01, 0.02, 0.03]) + assert data["I"].value == pytest.approx([100, 90, 80]) + + +# Qx wraps around at the end of each Qy row, as in 2D detector files. +GRID_DATA = "-0.1 -0.1 5\n0.0 -0.1 6\n0.1 -0.1 7\n-0.1 0.0 8\n0.0 0.0 9\n0.1 0.0 10\n" + + +def test_guess_params_with_guessed_dataset_type(tmp_path): + # Passing guess_dataset_type() uses the guessed type (as load_data_default_params does). + filename = tmp_path / "data.dat" + filename.write_text(GRID_DATA) + params = guess_params_from_filename(str(filename), guess_dataset_type(str(filename))) + assert params.dataset_type is two_dim + + # Tests for ignored columns (issue #263). IGNORE = ("", None) @@ -272,3 +300,60 @@ def test_column_without_unit_is_ignored(tmp_path): assert "dQ" not in data assert data["Q"].value == pytest.approx([0.01, 0.02]) assert data["I"].value == pytest.approx([100, 90]) + + +def test_dat_file_with_monotonic_q_is_guessed_as_1d(tmp_path): + filename = tmp_path / "data.dat" + filename.write_text(THREE_COLUMN_DATA) + assert guess_dataset_type(str(filename)) is one_dim + + +def test_non_dat_file_is_always_guessed_as_1d(tmp_path): + filename = tmp_path / "data.txt" + filename.write_text(GRID_DATA) + assert guess_dataset_type(str(filename)) is one_dim + + +@pytest.mark.parametrize( + "filename, expected", + [ + ("detector_square.dat", two_dim), + ("exp18_14_igor_2dqxqy.dat", two_dim), + ("nans_in_2d_data.DAT", two_dim), # has a blank line after every data row + ("nans_in_1d_data.dat", one_dim), + ], +) +def test_dat_dataset_type_guess_on_real_files(filename, expected): + assert guess_dataset_type(find(filename, "sasdataloader")) is expected + + +# Layout of an Anton Paar SAXSess export: a title, a line of integers and two +# lines of instrument parameters before the Q, I, dI rows. Classic Mac line endings. +SAXSESS_DATA = "\r".join( + [ + "", + "SAXS BOX", + " 4 30 0 0 0 0 0 0", + " 0.000000E+00 0.261200E+03 0.000000E+00 0.100000E+01 0.154200E+00", + " 0.100000E+01 0.155999E+01 0.000000E+00 0.000000E+00 0.000000E+00", + " 0.754396E-01 0.975440E+00 0.938981E+02", + " 0.150879E+00 0.975745E+00 0.991879E+02", + " 0.226319E+00 0.975473E+00 0.767819E+02", + " 0.301758E+00 0.974517E+00 0.131319E+03", + "", + ] +) + + +def test_saxsess_header_lines_are_not_read_as_data(tmp_path): + filename = tmp_path / "beam_profile.DAT" + filename.write_bytes(SAXSESS_DATA.encode()) + data = load_data_default_params(str(filename))[0] + assert data.dataset_type is one_dim + assert data["Q"].value == pytest.approx([0.0754396, 0.150879, 0.226319, 0.301758]) + assert data["I"].value == pytest.approx([0.975440, 0.975745, 0.975473, 0.974517]) + + +def test_blank_lines_between_rows_do_not_change_the_column_count(): + rows = [["0.01", "100", "1"], [""], ["0.02", "90", "2"], [""], ["0.03", "80", "3"], [""]] + assert guess_column_count(rows, 0) == 3