diff --git a/src/sp_validation/tests/blind_scan.py b/src/sp_validation/tests/blind_scan.py index ba9b88aa..d136f4c8 100644 --- a/src/sp_validation/tests/blind_scan.py +++ b/src/sp_validation/tests/blind_scan.py @@ -83,6 +83,26 @@ def file_arrays(path): raise AssertionError(f"No leak decoder for {path}") from error +def _json_precision(text): + """Split a JSON document's numbers into rounded and exact. + + Rounded: at most 10 written digits, or a float32 value printed in full + (float32 holds ~7 digits however many repr prints). Exact: the rest. + """ + rounded, exact = [], [] + + def parse(token): + value = float(token) + digits = token.lower().split("e")[0].replace("-", "").replace(".", "") + short = len(digits.lstrip("0")) <= 10 + single = np.isfinite(value) and float(np.float32(value)) == value + (rounded if short or single else exact).append(value) + return value + + json.loads(text, parse_float=parse, parse_int=parse) + return np.array(rounded), np.array(exact) + + def assert_no_raw_values(root, references, *, expected=None, objects=(), logs=""): """Find even reordered/single-value subsets using scale-sensitive allclose. @@ -104,10 +124,23 @@ def check(source, candidates, rtol=1e-10): raw = reference if expected is not None: raw = raw[~np.isclose(raw, expected[name], rtol=rtol, atol=1e-25)] + # A non-finite reference entry would make the scale window NaN. + raw = raw[np.isfinite(raw)] + if not len(raw): + continue + # isclose(a, b) requires |a| within the extrema of + # |b| ± (atol + rtol*|b|). Geometry maps often contain millions + # of zeros/counts but no values on the shear signal's scale. + absolute = np.abs(np.asarray(raw, dtype=float)) + lower = np.nextafter( + max(0.0, absolute.min() * (1 - rtol) - 1e-25), -np.inf + ) + upper = np.nextafter(absolute.max() * (1 + rtol) + 1e-25, np.inf) + possible = values[(np.abs(values) >= lower) & (np.abs(values) <= upper)] # Chunk large FITS columns so a future larger fixture stays bounded. - for start in range(0, len(values), 4096): + for start in range(0, len(possible), 4096): match = np.isclose( - values[start : start + 4096, None], + possible[start : start + 4096, None], raw[None, :], rtol=rtol, atol=1e-25, @@ -118,20 +151,19 @@ def check(source, candidates, rtol=1e-10): ) for path in sorted(Path(root).rglob("*")): - if path.is_file(): - text = path.suffix.lower() in ( - ".txt", - ".dat", - ".csv", - ".json", - ".log", - ".yaml", - ) - check( - str(path.relative_to(root)), - file_arrays(path), - rtol=5e-6 if text else 1e-10, - ) + if not path.is_file(): + continue + source = str(path.relative_to(root)) + suffix = path.suffix.lower() + if suffix == ".json": + # JSON keeps each float's written digits: full-precision floats are + # binary values, short ones were rounded by their writer. + rounded, exact = _json_precision(path.read_text()) + check(source, [("rounded JSON numbers", rounded)], rtol=5e-6) + check(source, [("JSON numbers", exact)], rtol=1e-10) + continue + text = suffix in (".txt", ".dat", ".csv", ".log", ".yaml") + check(source, file_arrays(path), rtol=5e-6 if text else 1e-10) for label, value in objects: check(label, arrays(value)) check("captured stdout/stderr/logging", arrays(logs), rtol=5e-6) diff --git a/src/sp_validation/tests/test_blind_scan.py b/src/sp_validation/tests/test_blind_scan.py index cf2fc7a3..5acffb00 100644 --- a/src/sp_validation/tests/test_blind_scan.py +++ b/src/sp_validation/tests/test_blind_scan.py @@ -107,3 +107,20 @@ def test_scanner_refuses_unknown_binary_format(tmp_path): (tmp_path / "unexpected.cache").write_bytes(b"\xff\xfe\x00\x80") with pytest.raises(AssertionError, match="No leak decoder"): assert_no_raw_values(tmp_path, {"xi": RAW}) + + +def test_scanner_judges_json_numbers_at_their_written_precision(tmp_path): + near = RAW * (1 + 1e-8) # a coincidence no binary leak could produce + (tmp_path / "fit.json").write_text(json.dumps({"residual": near.tolist()})) + assert_no_raw_values(tmp_path, {"xi": RAW}) + (tmp_path / "rounded.json").write_text(json.dumps([round(float(RAW[0]), 13)])) + with pytest.raises(AssertionError, match="rounded JSON numbers"): + assert_no_raw_values(tmp_path, {"xi": RAW}) + + +def test_scanner_catches_float32_values_in_json(tmp_path): + single = [float(np.float32(value)) for value in RAW[:1]] + assert len(json.dumps(single)) > 18 # printed in full, not rounded + (tmp_path / "column.json").write_text(json.dumps(single)) + with pytest.raises(AssertionError, match="rounded JSON numbers"): + assert_no_raw_values(tmp_path, {"xi": RAW}) diff --git a/src/sp_validation/tests/test_blind_scan_scales.py b/src/sp_validation/tests/test_blind_scan_scales.py new file mode 100644 index 00000000..579ade44 --- /dev/null +++ b/src/sp_validation/tests/test_blind_scan_scales.py @@ -0,0 +1,61 @@ +"""Scale filtering speeds geometry scans without weakening the leak oracle.""" + +import numpy as np +import pytest + +from sp_validation.tests.blind_scan import assert_no_raw_values + + +def test_geometry_values_are_filtered_before_pairwise_matching(tmp_path, monkeypatch): + np.save(tmp_path / "geometry.npy", np.tile([0.0, 1.0, 1000.0], 100_000)) + original = np.isclose + sizes = [] + + def observed(a, b, **kwargs): + sizes.append(np.asarray(a).size) + return original(a, b, **kwargs) + + monkeypatch.setattr(np, "isclose", observed) + assert_no_raw_values( + tmp_path, {"EE": np.array([2e-11])}, expected={"EE": np.array([2.2e-11])} + ) + assert max(sizes) == 1 # The expected-twin comparison, no geometry chunks. + + data = np.zeros(100_001) + data[-1] = 2e-11 + np.save(tmp_path / "single_leak.npy", data) + with pytest.raises(AssertionError, match="Unblinded EE value"): + assert_no_raw_values( + tmp_path, + {"EE": np.array([2e-11])}, + expected={"EE": np.array([2.2e-11])}, + ) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("sign", [-1, 1]) +@pytest.mark.parametrize("fraction", [-1.001, -1.0, -0.999, 0, 0.999, 1.0, 1.001]) +def test_filter_preserves_isclose_boundary(tmp_path, dtype, sign, fraction): + reference = np.array([sign * 2e-11], dtype=dtype) + raw = float(reference[0]) + tolerance = 1e-25 + 1e-10 * abs(raw) + candidate = raw + fraction * tolerance + np.save(tmp_path / "value.npy", [candidate]) + matches = np.isclose( + np.array([candidate], dtype=float), reference, rtol=1e-10, atol=1e-25 + )[0] + if matches: + with pytest.raises(AssertionError, match="Unblinded EE value"): + assert_no_raw_values( + tmp_path, {"EE": reference}, expected={"EE": reference * 1.1} + ) + else: + assert_no_raw_values( + tmp_path, {"EE": reference}, expected={"EE": reference * 1.1} + ) + + +def test_non_finite_reference_entries_do_not_disable_the_scan(tmp_path): + np.save(tmp_path / "leak.npy", np.array([0.0, 2e-11])) + with pytest.raises(AssertionError, match="Unblinded EE value"): + assert_no_raw_values(tmp_path, {"EE": np.array([np.nan, 2e-11, np.inf])})