v2 selection: upper size bound 9, FLAGS cut ignores bit 16

config/calibration/mask_v2.0.yaml

@@ -35,13 +35,14 @@ params:
 # Masks
 ## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
-  # SExtractor flags: keep 0 (clean), 1 (neighbours), 2 (deblended); drop
-  # 3 (both) and above. Same as the image-sims config, so data and sims
-  # select identically (shapepipe#885).
+  # Ignore bit 16 (incomplete aperture data), then keep FLAGS <= 2.
+  # Bit 16 preferentially flags large galaxies; selecting on it biases m in
+  # image sims. The image-sim validation uses this same predicate.
   - col_name: FLAGS
     label: SE FLAGS
     kind: smaller_equal
     value: 2
+    ignore_bits: 16
 
   # Duplicate objects
   - col_name: overlap
@@ -139,9 +140,9 @@ metacal:
   gal_snr_min: 10
   gal_snr_max: 500
 
-  # Relative-size (hlr / hlr_psf) range
+  # Moment ratio T_gal / T_PSF; admit galaxies below 9.
   gal_rel_size_min: 0.707
-  gal_rel_size_max: 3
+  gal_rel_size_max: 9
 
   # Correct relative size for ellipticity?
   gal_size_corr_ell: False

scripts/masking.py

@@ -79,7 +79,7 @@ def apply_masks(data, mask_config, footprint_only=False):
         kind = cut["kind"]
         value = cut["value"]
 
-        mask &= apply_condition(data[col], kind, value)
+        mask &= apply_condition(data[col], kind, value, cut.get("ignore_bits", 0))
 
     # --- metacal relative size (skip for footprint-only) ---
     if not footprint_only:

src/sp_validation/dashboard/selection_hist.py

@@ -7,6 +7,8 @@
 import numpy as np
 import yaml
 
+from sp_validation.masks import apply_condition
+
 
 def build(path, config, expected, chunk=500_000):
     """Apply recorded data cuts and the noshear metacal cuts cumulatively."""
@@ -35,21 +37,12 @@ def build(path, config, expected, chunk=500_000):
             mask = np.ones(len(rows), dtype=bool)
             stages = [mask.copy()]
             for cut in cuts:
-                x, v = rows[cut["col_name"]], cut["value"]
-                kind = cut["kind"]
-                if kind == "equal":
-                    keep = x == v
-                elif kind == "not_equal":
-                    keep = x != v
-                elif kind == "smaller_equal":
-                    keep = x <= v
-                elif kind == "greater_equal":
-                    keep = x >= v
-                elif kind == "range":
-                    keep = (x >= v[0]) & (x <= v[1])
-                else:
-                    raise ValueError(f"Unsupported selection {kind}")
-                mask &= keep
+                mask &= apply_condition(
+                    rows[cut["col_name"]],
+                    cut["kind"],
+                    cut["value"],
+                    ignore_bits=cut.get("ignore_bits", 0),
+                )
                 stages.append(mask.copy())
             mc = cfg["metacal"]
             with np.errstate(divide="ignore", invalid="ignore"):

src/sp_validation/masks.py

@@ -25,7 +25,7 @@
 }
 
 
-def apply_condition(array, kind, value):
+def apply_condition(array, kind, value, ignore_bits=0):
     """Apply Condition.
 
     Evaluate one mask condition (``kind``/``value``, as specified in mask
@@ -44,6 +44,9 @@ def apply_condition(array, kind, value):
     value : float or list
         value(s) to be used in mask operation; two-element list for "range"
 
+    ignore_bits : int, optional
+        Bits to clear before comparison; requires an integer array.
+
     Returns
     -------
     numpy.ndarray
@@ -53,6 +56,11 @@ def apply_condition(array, kind, value):
     kind = _KIND_ALIASES.get(kind, kind)
     if kind not in _KIND_OPS:
         raise ValueError(f"Unknown mask condition kind: {kind!r}")
+    if ignore_bits:
+        array = np.asarray(array)
+        if not np.issubdtype(array.dtype, np.integer):
+            raise ValueError("ignore_bits requires an integer array")
+        array = array & np.bitwise_not(np.array(ignore_bits, dtype=array.dtype))
     return _KIND_OPS[kind](array, value)
 
 
@@ -128,6 +136,8 @@ class Mask:
         input data, default is `None`; apply mask if given
     verbose : bool, optional
         verbose output if ``True``; default is ``False``
+    ignore_bits : int, optional
+        Bits to clear before evaluating the condition.
 
     """
 
@@ -140,7 +150,9 @@ def __init__(
         col_name2=None,
         dat=None,
         verbose=False,
+        ignore_bits=0,
     ):
+        self._ignore_bits = ignore_bits
         self._col_name = col_name
         self._label = label
         self._value = value
@@ -177,10 +189,14 @@ def from_list(cls, masks, label="combined", verbose=False):
     def apply(self, dat):
         if self._kind == "not_equal_2bands":
             self._mask = apply_condition(
-                dat[self._col_name], "not_equal", self._value
-            ) | apply_condition(dat[self._col_name2], "not_equal", self._value)
+                dat[self._col_name], "not_equal", self._value, self._ignore_bits
+            ) | apply_condition(
+                dat[self._col_name2], "not_equal", self._value, self._ignore_bits
+            )
         else:
-            self._mask = apply_condition(dat[self._col_name], self._kind, self._value)
+            self._mask = apply_condition(
+                dat[self._col_name], self._kind, self._value, self._ignore_bits
+            )
 
     def to_bool(self, hsp_mask):
         if self._verbose:
@@ -222,10 +238,14 @@ def get_sign(self, latex=False):
             sign = "$=$" if latex else "="
         elif self._kind == "not_equal":
             sign = r"$\ne$" if latex else "!="
-        elif self._kind in ("greater_equal", "range"):
-            sign = r"$\leq$" if latex else ">="
-        elif self._kind == "smaller_equal":
-            sign = r"$\geq$" if latex else "<="
+        elif self._kind == "greater_equal":
+            sign = r"$\geq$" if latex else ">="
+        elif self._kind in ("smaller_equal", "less_equal", "range"):
+            sign = r"$\leq$" if latex else "<="
+        elif self._kind == "greater":
+            sign = ">"
+        elif self._kind == "less":
+            sign = "<"
         return sign
 
     def print_condition(self, f_out, latex=False):
@@ -235,8 +255,10 @@ def print_condition(self, f_out, latex=False):
         sign = self.get_sign(latex=latex)
 
         name = self._label if latex else self._col_name
+        if self._ignore_bits:
+            name = f"({name} & ~{self._ignore_bits})"
 
-        if sign is not None:
+        if sign is not None and self._kind != "range":
             print(f"{name} {sign} {self._value}", file=f_out)
 
         if self._kind == "not_equal_2bands":
@@ -266,8 +288,13 @@ def create_descr(self):
         sign = self.get_sign()
         if sign is not None:
             descr = f"{sign}{self._value}"
+        if self._ignore_bits and sign is not None:
+            descr = f"({self._col_name} & ~{self._ignore_bits}) {sign} {self._value}"
         if self._kind == "range":
-            descr = f"{self._value[0]}<={self._col_name}<={self._value[1]}"
+            name = self._col_name
+            if self._ignore_bits:
+                name = f"({name} & ~{self._ignore_bits})"
+            descr = f"{self._value[0]}<={name}<={self._value[1]}"
         if self._kind == "not_equal_2bands":
             descr = f"!={self._value} in {self._col_name} or {self._col_name2}"
         self._descr = descr

src/sp_validation/tests/test_dashboard_selection.py

@@ -18,9 +18,11 @@ def test_cumulative_selection_and_expected_count(tmp_path):
         "NGMIX_T_NOSHEAR",
         "NGMIX_T_PSF_RECONV_NOSHEAR",
     ]
-    rows = np.zeros(4, dtype=[(name, "f8") for name in names])
+    rows = np.zeros(
+        4, dtype=[(name, "i4" if name == "FLAGS" else "f8") for name in names]
+    )
     rows["mag"] = [20, 21, 22, 23]
-    rows["FLAGS"] = [0, 0, 3, 0]
+    rows["FLAGS"] = [16, 0, 19, 0]
     rows["NGMIX_FLUX_NOSHEAR"] = [20, 5, 20, 20]
     rows["NGMIX_FLUX_ERR_NOSHEAR"] = 1
     rows["NGMIX_T_NOSHEAR"] = 1
@@ -38,6 +40,7 @@ def test_cumulative_selection_and_expected_count(tmp_path):
                         "label": "flags",
                         "kind": "smaller_equal",
                         "value": 2,
+                        "ignore_bits": 16,
                     }
                 ],
                 "metacal": {

src/sp_validation/tests/test_masks.py

@@ -52,6 +52,33 @@ def test_smaller_equal_alias_matches_less_equal():
     )
 
 
+@pytest.mark.parametrize("dtype", ["i4", "u4"])
+def test_ignore_bits_and_config_mask(dtype):
+    flags = np.array([0, 1, 2, 3, 16, 17, 18, 19, 20, 32], dtype=dtype)
+    expected = [True, True, True, False, True, True, True, False, False, False]
+    npt.assert_array_equal(apply_condition(flags, "less_equal", 2, 16), expected)
+    config = {
+        "dat": [
+            {
+                "col_name": "FLAGS",
+                "label": "flags",
+                "kind": "less_equal",
+                "value": 2,
+                "ignore_bits": 16,
+            }
+        ]
+    }
+    masks, _ = get_masks_from_config(config, {"FLAGS": flags})
+    npt.assert_array_equal(masks[0]._mask, expected)
+    masks[0].create_descr()
+    assert masks[0]._descr == "(FLAGS & ~16) <= 2"
+
+
+def test_ignore_bits_rejects_float():
+    with pytest.raises(ValueError, match="integer"):
+        apply_condition(np.array([1.0]), "less_equal", 2, 16)
+
+
 def test_unknown_kind_raises():
     with pytest.raises(ValueError):
         apply_condition(_ARRAY, "not_a_real_kind", 3)