fixed incorrect assignment of data type on file load #269

sasdata/data_io/importers/import_ascii.py

@@ -101,7 +101,7 @@ def guess_params_from_filename(filename: str, dataset_type: DatasetType) -> Asci
             columns,
             starting_line=startpos,
             separator_dict=separator_dict,
-            dataset_type=guess_dataset_type(filename),
+            dataset_type=dataset_type,
         )
         return params
 

sasdata/guess.py

@@ -1,11 +1,24 @@
+import re
+
+import numpy as np
 from scipy.stats import mode
 
 from sasdata.dataset_types import DatasetType, one_dim, two_dim
 
 
+def _is_blank(line: list[str]) -> bool:
+    return len(line) == 0 or line == [""]
+
+
 def guess_column_count(split_csv: list[list[str]], starting_pos: int) -> int:
-    """Guess the amount of columns present in the data."""
-    candidate_lines = split_csv[starting_pos::]
+    """Guess the amount of columns present in the data.
+
+    Blank lines are not counted, so files with an empty line between data rows
+    are not guessed as having a single column.
+    """
+    candidate_lines = [line for line in split_csv[starting_pos::] if not _is_blank(line)]
+    if not candidate_lines:
+        return 0
     return int(mode([len(line) for line in candidate_lines]).mode)
 
 
@@ -34,29 +47,69 @@ def guess_columns(col_count: int, dataset_type: DatasetType) -> list[str]:
 symbols_to_ignore = [".", "-", "+", "e", "E"]
 
 
+def _is_numeric_row(row: list[str]) -> bool:
+    for column in row:
+        amended_column = column
+        for symbol in symbols_to_ignore:
+            amended_column = amended_column.replace(symbol, "")
+        if not amended_column.isdigit():
+            return False
+    return True
+
+
 def guess_starting_position(split_csv: list[list[str]]) -> int:
-    """Try to look for a line where the first item in the row can be converted
-    to a number. If such a line doesn't exist, try to look for a line where the
-    first item in the row can be converted to a number. If such a line doesn't
-    exist, then just return 0 as the starting position.
+    """Find the first line of the data block.
+
+    The data block is made of numeric lines with the most common column count.
+    Numeric header lines with a different column count (for example the
+    parameter lines of an Anton Paar SAXSess file) are skipped. If no line is
+    numeric, then just return 0 as the starting position.
 
     """
-    for i, row in enumerate(split_csv):
-        all_nums = True
-        for column in row:
-            amended_column = column
-            for symbol in symbols_to_ignore:
-                amended_column = amended_column.replace(symbol, "")
-            if not amended_column.isdigit():
-                all_nums = False
-                break
-        if all_nums:
-            return i
-    return 0
-
-
-def guess_dataset_type(filename: str) -> DatasetType:
-    if filename.lower().endswith(".dat"):
+    numeric_rows = [(i, len(row)) for i, row in enumerate(split_csv) if not _is_blank(row) and _is_numeric_row(row)]
+    if not numeric_rows:
+        return 0
+    data_column_count = int(mode([count for _, count in numeric_rows]).mode)
+    return next(i for i, count in numeric_rows if count == data_column_count)
+
+
+def _split_all_separators(line: str) -> list[str]:
+    # Same splitting as the ASCII importer with every separator enabled.
+    return re.split(r"\s+|,|\t", line.strip())
+
+
+def guess_dataset_type(filename: str, split_csv: list[list[str]] | None = None) -> DatasetType:
+    """Guess whether an ASCII file holds 1D or 2D data.
+
+    Only ``.dat`` files can be 2D. They are guessed from their contents: 2D data
+    has at least three columns, and its first column (Qx) repeats or wraps
+    around, while the Q column of 1D data is strictly monotonic.
+
+    :param filename: The file to guess for.
+    :param split_csv: The file's lines, already split. Read from ``filename`` if not given.
+    """
+    if not filename.lower().endswith(".dat"):
+        return one_dim
+    if split_csv is None:
+        with open(filename) as file:
+            split_csv = [_split_all_separators(line) for line in file.readlines()]
+    start = guess_starting_position(split_csv)
+    column_count = guess_column_count(split_csv, start)
+    if column_count < 3:
+        return one_dim
+    first_column = []
+    for row in split_csv[start:]:
+        if len(row) < column_count:
+            continue
+        try:
+            first_column.append(float(row[0]))
+        except ValueError:
+            continue
+    first_column = np.array(first_column)
+    first_column = first_column[np.isfinite(first_column)]
+    if len(first_column) < 2:
         return two_dim
-    else:
+    steps = np.diff(first_column)
+    if np.all(steps > 0) or np.all(steps < 0):
         return one_dim
+    return two_dim

test/data_io/utest_import_ascii.py

@@ -9,8 +9,8 @@
     load_data,
     load_data_default_params,
 )
-from sasdata.dataset_types import one_dim
-from sasdata.guess import guess_columns
+from sasdata.dataset_types import one_dim, two_dim
+from sasdata.guess import guess_column_count, guess_columns, guess_dataset_type
 from sasdata.quantities.units import per_angstrom, per_centimeter
 from test import local_data_finder, local_mumag_finder
 
@@ -251,6 +251,34 @@ def test_extra_columns_are_ignored(tmp_path):
     assert data["I"].value == pytest.approx([100, 90])
 
 
+THREE_COLUMN_DATA = "0.01 100 1\n0.02 90 2\n0.03 80 3\n"
+
+
+def test_guess_params_honours_requested_dataset_type(tmp_path):
+    # guess_dataset_type() classifies every .dat file as 2D. An explicit request
+    # for 1D must win, otherwise the 1D columns are checked against the 2D type.
+    filename = tmp_path / "data.dat"
+    filename.write_text(THREE_COLUMN_DATA)
+    params = guess_params_from_filename(str(filename), one_dim)
+    assert params.dataset_type is one_dim
+    data = load_data(params)[0]
+    assert data.dataset_type is one_dim
+    assert data["Q"].value == pytest.approx([0.01, 0.02, 0.03])
+    assert data["I"].value == pytest.approx([100, 90, 80])
+
+
+# Qx wraps around at the end of each Qy row, as in 2D detector files.
+GRID_DATA = "-0.1 -0.1 5\n0.0 -0.1 6\n0.1 -0.1 7\n-0.1 0.0 8\n0.0 0.0 9\n0.1 0.0 10\n"
+
+
+def test_guess_params_with_guessed_dataset_type(tmp_path):
+    # Passing guess_dataset_type() uses the guessed type (as load_data_default_params does).
+    filename = tmp_path / "data.dat"
+    filename.write_text(GRID_DATA)
+    params = guess_params_from_filename(str(filename), guess_dataset_type(str(filename)))
+    assert params.dataset_type is two_dim
+
+
 # Tests for ignored columns (issue #263).
 
 IGNORE = ("<ignore>", None)
@@ -272,3 +300,60 @@ def test_column_without_unit_is_ignored(tmp_path):
     assert "dQ" not in data
     assert data["Q"].value == pytest.approx([0.01, 0.02])
     assert data["I"].value == pytest.approx([100, 90])
+
+
+def test_dat_file_with_monotonic_q_is_guessed_as_1d(tmp_path):
+    filename = tmp_path / "data.dat"
+    filename.write_text(THREE_COLUMN_DATA)
+    assert guess_dataset_type(str(filename)) is one_dim
+
+
+def test_non_dat_file_is_always_guessed_as_1d(tmp_path):
+    filename = tmp_path / "data.txt"
+    filename.write_text(GRID_DATA)
+    assert guess_dataset_type(str(filename)) is one_dim
+
+
+@pytest.mark.parametrize(
+    "filename, expected",
+    [
+        ("detector_square.dat", two_dim),
+        ("exp18_14_igor_2dqxqy.dat", two_dim),
+        ("nans_in_2d_data.DAT", two_dim),  # has a blank line after every data row
+        ("nans_in_1d_data.dat", one_dim),
+    ],
+)
+def test_dat_dataset_type_guess_on_real_files(filename, expected):
+    assert guess_dataset_type(find(filename, "sasdataloader")) is expected
+
+
+# Layout of an Anton Paar SAXSess export: a title, a line of integers and two
+# lines of instrument parameters before the Q, I, dI rows. Classic Mac line endings.
+SAXSESS_DATA = "\r".join(
+    [
+        "",
+        "SAXS BOX",
+        "       4        30         0         0         0         0         0         0",
+        "  0.000000E+00   0.261200E+03   0.000000E+00   0.100000E+01   0.154200E+00",
+        "  0.100000E+01   0.155999E+01   0.000000E+00   0.000000E+00   0.000000E+00",
+        "  0.754396E-01   0.975440E+00   0.938981E+02",
+        "  0.150879E+00   0.975745E+00   0.991879E+02",
+        "  0.226319E+00   0.975473E+00   0.767819E+02",
+        "  0.301758E+00   0.974517E+00   0.131319E+03",
+        "",
+    ]
+)
+
+
+def test_saxsess_header_lines_are_not_read_as_data(tmp_path):
+    filename = tmp_path / "beam_profile.DAT"
+    filename.write_bytes(SAXSESS_DATA.encode())
+    data = load_data_default_params(str(filename))[0]
+    assert data.dataset_type is one_dim
+    assert data["Q"].value == pytest.approx([0.0754396, 0.150879, 0.226319, 0.301758])
+    assert data["I"].value == pytest.approx([0.975440, 0.975745, 0.975473, 0.974517])
+
+
+def test_blank_lines_between_rows_do_not_change_the_column_count():
+    rows = [["0.01", "100", "1"], [""], ["0.02", "90", "2"], [""], ["0.03", "80", "3"], [""]]
+    assert guess_column_count(rows, 0) == 3