Workflow: isolated catalogue registries, shear-only catalogues, narrower rerun triggers

src/sp_validation/cosmo_val/core.py

@@ -584,6 +584,14 @@ def init_results(self, objectwise=False):
 
         results = {}
         for ver in self.versions:
+            if objectwise and not all(
+                key in self.cc[ver]["shear"] for key in ("e1_PSF_col", "e2_PSF_col")
+            ):
+                self.print_magenta(
+                    f"Skipping object-wise leakage for {ver}: "
+                    "no galaxy PSF ellipticity columns configured"
+                )
+                continue
             obj = results[ver] = make_leakage()
             obj.entries = self.cc[ver]
             obj._params.update(set_params(ver))

src/sp_validation/cosmo_val/diagnostic_plots.py

@@ -45,6 +45,15 @@ def plot_objectwise_leakage(coefficients, styles, output):
         "alpha_0": (r"$\alpha(0)$", ":", "left"),
     }
     handles = []
+    if not coefficients:
+        ax.text(
+            0.5,
+            0.5,
+            "No object-wise leakage coefficients available",
+            ha="center",
+            va="center",
+            transform=ax.transAxes,
+        )
     for ver, bins in coefficients.items():
         style = styles[ver]
         for coeff in bins.values():

src/sp_validation/tests/test_cosmo_val_optional_psf.py

@@ -0,0 +1,23 @@
+"""Catalogues without galaxy PSF shapes still support PSF cross-correlations."""
+
+from types import SimpleNamespace
+
+from sp_validation.cosmo_val.core import CosmologyValidation
+from sp_validation.cosmo_val.diagnostic_plots import plot_objectwise_leakage
+
+
+def test_missing_galaxy_psf_skips_objectwise_regression(tmp_path):
+    messages = []
+    cv = SimpleNamespace(
+        versions=["mock"],
+        cc={"mock": {"shear": {"e1_col": "e1", "e2_col": "e2"}}},
+        print_magenta=messages.append,
+        # The builder must not be called for an inapplicable regression.
+        set_params_leakage_object=lambda _: (_ for _ in ()).throw(AssertionError()),
+        set_params_leakage_scale=None,
+    )
+    assert CosmologyValidation.init_results(cv, objectwise=True) == {}
+    assert "no galaxy PSF ellipticity columns" in messages[0]
+    output = tmp_path / "unavailable.png"
+    plot_objectwise_leakage({}, {}, output)
+    assert output.is_file()

workflow/README.md

@@ -146,10 +146,13 @@ This is the default because the alternative is incoherent: Snakemake's
 rule executes new script code against an old `import sp_validation` — the two
 halves of one commit, split.
 
-The catalogue config is the launched checkout's `cosmo_val/cat_config.yaml`.
-Pseudo-Cℓ estimation defaults to the catalogue estimator. Set
-`cosmo_val.cell_method: map` in the run config to select the retained
-map-based estimator; the catalogue estimator remains the default when omitted.
+The catalogue config defaults to the launched checkout's `cosmo_val/cat_config.yaml`.
+Set `catalog_config: /path/to/catalogues.yaml` in the run config to use an isolated
+registry, including its own `paths.blinds`; every producer reads that same file.
+Configured catalogue names also work as version wildcards, so a mock entry does
+not need a ShapePipe `SP_v...` name.
+Pseudo-Cℓ estimation defaults to the catalogue estimator.
+Set `cosmo_val.cell_method: map` in the run config to select the map estimator.
 `COSMO_VAL` defaults to the launched checkout's `cosmo_val/output`, so writing
 into another checkout's products means naming it; `COSMO_INFERENCE` defaults to
 the shared candide tree, which holds the covariances and only its owner

workflow/common.py

@@ -81,8 +81,8 @@ def _load_checkout_module(name):
         "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference"
     )
 )
-# The catalogue config of the launched checkout: the one file both the host
-# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from.
+# The default catalogue config. configure() resolves a run's catalog_config
+# override so the host and every job read the same file.
 CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml")
 
 # Fiducial cosmology: Planck 2018 (astropy Planck18, Table 2 + BAO)
@@ -196,12 +196,30 @@ def warn_if_image_stale():
 
 def configure(workflow_config):
     """Install config-derived values after Snakemake has loaded configfiles."""
-    global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18
+    global CAT_CONFIG, CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18
     from snakemake.common.configfile import load_configfile
 
     inject_checkout_pythonpath(workflow_config)
     warn_if_image_stale()
+    CAT_CONFIG = str(
+        _plain(
+            workflow_config.get(
+                "catalog_config", REPO_ROOT / "cosmo_val" / "cat_config.yaml"
+            )
+        )
+    )
     CATALOG_CONFIG = load_configfile(CAT_CONFIG)
+    # Explicit catalogue names support mock and external-survey entries without
+    # allowing a version wildcard to consume the binning suffix of an output.
+    names = [
+        re.escape(name)
+        for name, entry in CATALOG_CONFIG.items()
+        if isinstance(entry, dict) and "shear" in entry
+    ]
+    WILDCARD_CONSTRAINTS["version"] = (
+        r"SP_v[\d.]+(_[ABC])?(_w_iv)?(_ecut\d+)?(_leak_corr)?"
+        + ("|" + "|".join(names) if names else "")
+    )
     FIDUCIAL = workflow_config["fiducial"]
     DEFAULT_MASK_SUFFIX = (
         "_masked" if workflow_config["covariance"].get("default_masked", False) else ""
@@ -332,6 +350,45 @@ def catalogue_entry(version):
     return CATALOG_CONFIG[catalogue_name(version)]
 
 
+def measurement_catalogue(version, *, psf=False):
+    """Load-bearing catalogue metadata for two-point producer invalidation.
+
+    Styles and precomputed survey summaries are presentation/covariance inputs,
+    not inputs to these measurements. Keep them out of expensive rerun triggers.
+    The PSF producers additionally read the stellar entries and patch count.
+    """
+    entry = catalogue_entry(version)
+    shear_keys = (
+        "path",
+        "hdu",
+        "column_map",
+        "ra_col",
+        "dec_col",
+        "w_col",
+        "e1_col",
+        "e2_col",
+        "e1_col_corrected",
+        "e2_col_corrected",
+        "R",
+        "R11",
+        "R22",
+        "redshift_path",
+        "tomo_bin_col",
+    )
+    metadata = {
+        "subdir": entry.get("subdir", ""),
+        "shear": {
+            key: entry["shear"][key] for key in shear_keys if key in entry["shear"]
+        },
+        "mask": entry.get("mask"),
+    }
+    if psf:
+        metadata.update(
+            {key: entry[key] for key in ("psf", "star", "patch_number") if key in entry}
+        )
+    return metadata
+
+
 def redshift_path(version):
     """The n(z) file of ``version``: its catalogue entry's ``shear.redshift_path``,
     as written (as ``CosmologyValidation.get_redshift`` reads it)."""

workflow/rules/twopoint.smk

@@ -36,6 +36,7 @@ rule xi:
         output_dir=str(COSMO_VAL),
         grid=lambda w: grid_of(XI_GRIDS, w),
         b_target=config.get("cosmo_val", {}).get("b_target", 0.01),
+        catalogue=lambda w: measurement_catalogue(w.version),
         blind=lambda w: blind_of(w.version),
     resources:
         # The fine integration grid needs more memory and wall time than the
@@ -98,6 +99,7 @@ rule rho_tau_stats:
         cat_config=CAT_CONFIG,
         output_dir=str(COSMO_VAL),
         b_target=config.get("cosmo_val", {}).get("b_target", 0.01),
+        catalogue=lambda w: measurement_catalogue(w.version, psf=True),
         blind=lambda w: blind_of(w.version),
     resources:
         mem_mb=64000,
@@ -146,6 +148,7 @@ rule pseudo_cl:
         binning="{binning}",
         nbins=lambda w: int(w.nbins),
         power=0.5,
+        catalogue=lambda w: measurement_catalogue(w.version),
         compute_tomography=lambda w: w.tomo_suffix == "_tomo",
         blind=lambda w: blind_of(w.version),
     resources:
@@ -169,6 +172,7 @@ rule pseudo_cl_cov:
         binning="{binning}",
         nbins=lambda w: int(w.nbins),
         power=0.5,
+        catalogue=lambda w: measurement_catalogue(w.version),
         compute_tomography=lambda w: w.tomo_suffix == "_tomo",
     resources:
         mem_mb=16000,

workflow/tests/test_catalog_config.py

@@ -0,0 +1,77 @@
+"""Run-specific catalogue registries reach producers, including mock names."""
+
+import yaml
+from conftest import parse_jobs
+
+
+def test_external_mock_registry(toy, tmp_path):
+    name = "GLASS_00001_blinded"
+    catalogue = tmp_path / "mock.fits"
+    catalogue.touch()
+    registry = tmp_path / "catalogues.yaml"
+    registry.write_text(
+        yaml.safe_dump(
+            {
+                name: {
+                    "blind": "mock-test",
+                    "colour": "C0",
+                    "marker": "o",
+                    "subdir": str(tmp_path),
+                    "shear": {
+                        "path": str(catalogue),
+                        "redshift_path": str(tmp_path / "nz.txt"),
+                    },
+                },
+                "paths": {"blinds": str(tmp_path / "mock-blinds")},
+            }
+        )
+    )
+    target = str(
+        toy.common._plain(toy.cosmo_val)
+        / (f"{name}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100.sacc")
+    )
+    result = toy.snakemake(
+        "-n",
+        target,
+        config=[
+            f"catalog_config={registry}",
+            f'versions=["{name}"]',
+            f'fiducial={{"version":"{name}","mock_version":"{name}"}}',
+        ],
+    )
+    assert result.returncode == 0, result.stdout
+    (job,) = [job for job in parse_jobs(result.stdout) if job.rule == "xi"]
+    assert [toy.common._plain(p) for p in job.input] == [toy.common._plain(catalogue)]
+    assert job.wildcards["version"] == name
+
+    # Record producer params without running a scientific measurement.
+    from pathlib import Path
+
+    Path(target).parent.mkdir(parents=True, exist_ok=True)
+    Path(target).touch()
+    overrides = [
+        f"catalog_config={registry}",
+        f'versions=["{name}"]',
+        f'fiducial={{"version":"{name}","mock_version":"{name}"}}',
+    ]
+    touched = toy.snakemake("--touch", "--force", target, config=overrides)
+    assert touched.returncode == 0, touched.stdout
+    unchanged = toy.snakemake("-n", target, config=overrides)
+    assert unchanged.returncode == 0, unchanged.stdout
+    assert not parse_jobs(unchanged.stdout)
+
+    entries = yaml.safe_load(registry.read_text())
+    entries[name]["colour"] = "C3"
+    entries[name]["marker"] = "s"
+    entries[name]["cov_th"] = {"A": 1234.0, "sigma_e": 0.27}
+    registry.write_text(yaml.safe_dump(entries))
+    restyled = toy.snakemake("-n", target, config=overrides)
+    assert restyled.returncode == 0, restyled.stdout
+    assert not parse_jobs(restyled.stdout)
+
+    entries[name]["shear"]["redshift_path"] = str(tmp_path / "other_nz.txt")
+    registry.write_text(yaml.safe_dump(entries))
+    changed = toy.snakemake("-n", target, config=overrides)
+    assert changed.returncode == 0, changed.stdout
+    assert [job.rule for job in parse_jobs(changed.stdout)] == ["xi"]
+    assert "params have changed" in changed.stdout.lower()