From f67322d88a9853e8dbb93dffc137d32de40eea7d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 30 Oct 2026 03:37:02 +0100 Subject: [PATCH 1/2] Support ignored bits in catalogue selection cuts --- scripts/masking.py | 2 +- src/sp_validation/dashboard/selection_hist.py | 23 ++++----- src/sp_validation/masks.py | 47 +++++++++++++++---- .../tests/test_dashboard_selection.py | 7 ++- src/sp_validation/tests/test_masks.py | 27 +++++++++++ 5 files changed, 78 insertions(+), 28 deletions(-) diff --git a/scripts/masking.py b/scripts/masking.py index c3acbecde..20a4e8905 100644 --- a/scripts/masking.py +++ b/scripts/masking.py @@ -79,7 +79,7 @@ def apply_masks(data, mask_config, footprint_only=False): kind = cut["kind"] value = cut["value"] - mask &= apply_condition(data[col], kind, value) + mask &= apply_condition(data[col], kind, value, cut.get("ignore_bits", 0)) # --- metacal relative size (skip for footprint-only) --- if not footprint_only: diff --git a/src/sp_validation/dashboard/selection_hist.py b/src/sp_validation/dashboard/selection_hist.py index 9c584fd3c..ddc2c79f0 100644 --- a/src/sp_validation/dashboard/selection_hist.py +++ b/src/sp_validation/dashboard/selection_hist.py @@ -7,6 +7,8 @@ import numpy as np import yaml +from sp_validation.masks import apply_condition + def build(path, config, expected, chunk=500_000): """Apply recorded data cuts and the noshear metacal cuts cumulatively.""" @@ -35,21 +37,12 @@ def build(path, config, expected, chunk=500_000): mask = np.ones(len(rows), dtype=bool) stages = [mask.copy()] for cut in cuts: - x, v = rows[cut["col_name"]], cut["value"] - kind = cut["kind"] - if kind == "equal": - keep = x == v - elif kind == "not_equal": - keep = x != v - elif kind == "smaller_equal": - keep = x <= v - elif kind == "greater_equal": - keep = x >= v - elif kind == "range": - keep = (x >= v[0]) & (x <= v[1]) - else: - raise ValueError(f"Unsupported selection {kind}") - mask &= keep + mask &= apply_condition( + rows[cut["col_name"]], + cut["kind"], + cut["value"], + ignore_bits=cut.get("ignore_bits", 0), + ) stages.append(mask.copy()) mc = cfg["metacal"] with np.errstate(divide="ignore", invalid="ignore"): diff --git a/src/sp_validation/masks.py b/src/sp_validation/masks.py index 375145083..6c3ca0f35 100644 --- a/src/sp_validation/masks.py +++ b/src/sp_validation/masks.py @@ -25,7 +25,7 @@ } -def apply_condition(array, kind, value): +def apply_condition(array, kind, value, ignore_bits=0): """Apply Condition. Evaluate one mask condition (``kind``/``value``, as specified in mask @@ -44,6 +44,9 @@ def apply_condition(array, kind, value): value : float or list value(s) to be used in mask operation; two-element list for "range" + ignore_bits : int, optional + Bits to clear before comparison; requires an integer array. + Returns ------- numpy.ndarray @@ -53,6 +56,11 @@ def apply_condition(array, kind, value): kind = _KIND_ALIASES.get(kind, kind) if kind not in _KIND_OPS: raise ValueError(f"Unknown mask condition kind: {kind!r}") + if ignore_bits: + array = np.asarray(array) + if not np.issubdtype(array.dtype, np.integer): + raise ValueError("ignore_bits requires an integer array") + array = array & np.bitwise_not(np.array(ignore_bits, dtype=array.dtype)) return _KIND_OPS[kind](array, value) @@ -128,6 +136,8 @@ class Mask: input data, default is `None`; apply mask if given verbose : bool, optional verbose output if ``True``; default is ``False`` + ignore_bits : int, optional + Bits to clear before evaluating the condition. """ @@ -140,7 +150,9 @@ def __init__( col_name2=None, dat=None, verbose=False, + ignore_bits=0, ): + self._ignore_bits = ignore_bits self._col_name = col_name self._label = label self._value = value @@ -177,10 +189,14 @@ def from_list(cls, masks, label="combined", verbose=False): def apply(self, dat): if self._kind == "not_equal_2bands": self._mask = apply_condition( - dat[self._col_name], "not_equal", self._value - ) | apply_condition(dat[self._col_name2], "not_equal", self._value) + dat[self._col_name], "not_equal", self._value, self._ignore_bits + ) | apply_condition( + dat[self._col_name2], "not_equal", self._value, self._ignore_bits + ) else: - self._mask = apply_condition(dat[self._col_name], self._kind, self._value) + self._mask = apply_condition( + dat[self._col_name], self._kind, self._value, self._ignore_bits + ) def to_bool(self, hsp_mask): if self._verbose: @@ -222,10 +238,14 @@ def get_sign(self, latex=False): sign = "$=$" if latex else "=" elif self._kind == "not_equal": sign = r"$\ne$" if latex else "!=" - elif self._kind in ("greater_equal", "range"): - sign = r"$\leq$" if latex else ">=" - elif self._kind == "smaller_equal": - sign = r"$\geq$" if latex else "<=" + elif self._kind == "greater_equal": + sign = r"$\geq$" if latex else ">=" + elif self._kind in ("smaller_equal", "less_equal", "range"): + sign = r"$\leq$" if latex else "<=" + elif self._kind == "greater": + sign = ">" + elif self._kind == "less": + sign = "<" return sign def print_condition(self, f_out, latex=False): @@ -235,8 +255,10 @@ def print_condition(self, f_out, latex=False): sign = self.get_sign(latex=latex) name = self._label if latex else self._col_name + if self._ignore_bits: + name = f"({name} & ~{self._ignore_bits})" - if sign is not None: + if sign is not None and self._kind != "range": print(f"{name} {sign} {self._value}", file=f_out) if self._kind == "not_equal_2bands": @@ -266,8 +288,13 @@ def create_descr(self): sign = self.get_sign() if sign is not None: descr = f"{sign}{self._value}" + if self._ignore_bits and sign is not None: + descr = f"({self._col_name} & ~{self._ignore_bits}) {sign} {self._value}" if self._kind == "range": - descr = f"{self._value[0]}<={self._col_name}<={self._value[1]}" + name = self._col_name + if self._ignore_bits: + name = f"({name} & ~{self._ignore_bits})" + descr = f"{self._value[0]}<={name}<={self._value[1]}" if self._kind == "not_equal_2bands": descr = f"!={self._value} in {self._col_name} or {self._col_name2}" self._descr = descr diff --git a/src/sp_validation/tests/test_dashboard_selection.py b/src/sp_validation/tests/test_dashboard_selection.py index 484196e7b..40290f549 100644 --- a/src/sp_validation/tests/test_dashboard_selection.py +++ b/src/sp_validation/tests/test_dashboard_selection.py @@ -18,9 +18,11 @@ def test_cumulative_selection_and_expected_count(tmp_path): "NGMIX_T_NOSHEAR", "NGMIX_T_PSF_RECONV_NOSHEAR", ] - rows = np.zeros(4, dtype=[(name, "f8") for name in names]) + rows = np.zeros( + 4, dtype=[(name, "i4" if name == "FLAGS" else "f8") for name in names] + ) rows["mag"] = [20, 21, 22, 23] - rows["FLAGS"] = [0, 0, 3, 0] + rows["FLAGS"] = [16, 0, 19, 0] rows["NGMIX_FLUX_NOSHEAR"] = [20, 5, 20, 20] rows["NGMIX_FLUX_ERR_NOSHEAR"] = 1 rows["NGMIX_T_NOSHEAR"] = 1 @@ -38,6 +40,7 @@ def test_cumulative_selection_and_expected_count(tmp_path): "label": "flags", "kind": "smaller_equal", "value": 2, + "ignore_bits": 16, } ], "metacal": { diff --git a/src/sp_validation/tests/test_masks.py b/src/sp_validation/tests/test_masks.py index 840ae2592..60adc12a5 100644 --- a/src/sp_validation/tests/test_masks.py +++ b/src/sp_validation/tests/test_masks.py @@ -52,6 +52,33 @@ def test_smaller_equal_alias_matches_less_equal(): ) +@pytest.mark.parametrize("dtype", ["i4", "u4"]) +def test_ignore_bits_and_config_mask(dtype): + flags = np.array([0, 1, 2, 3, 16, 17, 18, 19, 20, 32], dtype=dtype) + expected = [True, True, True, False, True, True, True, False, False, False] + npt.assert_array_equal(apply_condition(flags, "less_equal", 2, 16), expected) + config = { + "dat": [ + { + "col_name": "FLAGS", + "label": "flags", + "kind": "less_equal", + "value": 2, + "ignore_bits": 16, + } + ] + } + masks, _ = get_masks_from_config(config, {"FLAGS": flags}) + npt.assert_array_equal(masks[0]._mask, expected) + masks[0].create_descr() + assert masks[0]._descr == "(FLAGS & ~16) <= 2" + + +def test_ignore_bits_rejects_float(): + with pytest.raises(ValueError, match="integer"): + apply_condition(np.array([1.0]), "less_equal", 2, 16) + + def test_unknown_kind_raises(): with pytest.raises(ValueError): apply_condition(_ARRAY, "not_a_real_kind", 3) From 534f7ea47d730c09d4ce2ca8daa4637512447b39 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 30 Oct 2026 03:37:03 +0100 Subject: [PATCH 2/2] Adopt v2 size bound 9 and ignore FLAGS bit 16 --- config/calibration/mask_v2.0.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/config/calibration/mask_v2.0.yaml b/config/calibration/mask_v2.0.yaml index cb89336b8..ae8a71769 100644 --- a/config/calibration/mask_v2.0.yaml +++ b/config/calibration/mask_v2.0.yaml @@ -35,13 +35,14 @@ params: # Masks ## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: - # SExtractor flags: keep 0 (clean), 1 (neighbours), 2 (deblended); drop - # 3 (both) and above. Same as the image-sims config, so data and sims - # select identically (shapepipe#885). + # Ignore bit 16 (incomplete aperture data), then keep FLAGS <= 2. + # Bit 16 preferentially flags large galaxies; selecting on it biases m in + # image sims. The image-sim validation uses this same predicate. - col_name: FLAGS label: SE FLAGS kind: smaller_equal value: 2 + ignore_bits: 16 # Duplicate objects - col_name: overlap @@ -139,9 +140,9 @@ metacal: gal_snr_min: 10 gal_snr_max: 500 - # Relative-size (hlr / hlr_psf) range + # Moment ratio T_gal / T_PSF; admit galaxies below 9. gal_rel_size_min: 0.707 - gal_rel_size_max: 3 + gal_rel_size_max: 9 # Correct relative size for ellipticity? gal_size_corr_ell: False