From 7255a0eeba4f0e7300cfb87451da500e51743b02 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:17:19 +0200 Subject: [PATCH 01/36] image_sims: add pipeline_config_dir to swap the ngmix ini per run The ShapePipe pipeline stage's ngmix ini (which owns METACAL_PSF) is resolved from the shapepipe worktree's example/cfis_image_sims via run_job's self-computed config_dir, so nothing in the run config could select a different ini -- an A/B on the metacal reconvolution PSF needed a separate worktree per arm. Add an optional operational key `pipeline_config_dir`. When set it is passed to run_job as `-c DIR`, redirecting the pipeline stage's whole ShapePipe config tree (ngmix ini included) at a per-arm copy; when empty (the default) no `-c` is passed and run_job self-computes exactly as before, so runs predating this key reproduce bit-exact. Merge/final_cat still read the top-level cfis symlink (config_dir), untouched. Candidate upstream PR for the image-sims orchestration branch. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XSWPiisH6y7NFFbjHqthuC --- workflow/image_sims/config.yaml | 5 +++++ workflow/rules/image_sims.smk | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 8f719c68..144b52b3 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -63,6 +63,11 @@ image_sims: # ShapePipe cfis configs (final_cat.param etc.); default is # {shapepipe_repo}/example/cfis_image_sims once #766 lands. [operational] config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims + # ShapePipe config tree for the pipeline stage (run_job's -c override); owns + # the ngmix ini and hence METACAL_PSF. Empty -> run_job's default, + # {shapepipe_repo}/example/cfis_image_sims. Point it at a per-arm copy to + # A/B the metacal PSF without editing the worktree. [operational] + pipeline_config_dir: "" psf_model: psfex # [operational] n_smp: -1 # [operational] # Extract/calibrate scripts run from the sp_validation repo checkout (branch diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index a4eb92d0..4f2c2fed 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -86,6 +86,7 @@ _OPERATIONAL_KEYS = { "branches", "shape", "config_dir", + "pipeline_config_dir", "psf_model", "n_smp", "extract_script", @@ -168,6 +169,10 @@ MASK_CONFIG = IMSIM["mask_config"] # e.g. config/calibration/mask_v1.X.9_im_sim PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py" # ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param). CONFIG_DIR = IMSIM["config_dir"] +# ShapePipe config dir for the pipeline stage's run_job (``-c`` override; owns +# the ngmix ini and thus METACAL_PSF). Empty -> run_job's default. See +# config.yaml. +PIPELINE_CONFIG_DIR = IMSIM["pipeline_config_dir"] # ShapePipe scripts live in the ShapePipe repo (also baked into its image). CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" @@ -372,6 +377,7 @@ rule im_pipeline: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", psf=IMSIM["psf_model"], n_smp=IMSIM["n_smp"], + config_flag=f"-c {PIPELINE_CONFIG_DIR}" if PIPELINE_CONFIG_DIR else "", resources: mem_mb=16000, runtime=720, @@ -379,7 +385,7 @@ rule im_pipeline: "cd {params.run_dir} && " "{EXEC_PIPELINE} bash {RUN_JOB} " "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " - "-p {params.psf} -N {params.n_smp}" + "-p {params.psf} -N {params.n_smp} {params.config_flag}" rule im_merge: From 77fc09e15d7bb5908e640c6370bb21816decaa0b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 19 Jul 2026 22:34:46 +0200 Subject: [PATCH 02/36] fix(image-sims): honour sims_type for m-bias sim-dir naming ImageSimMBias.load_catalogs hardcoded the '_grid_' infix between branch name and run number, so blended arms ({branch}_{num} dirs) hit FileNotFoundError on a '{branch}_grid_{num}' path that never existed. Derive the suffix from sims_type (defaulting to grid for backward compat and the synthetic-recovery tests), mirroring the workflow's own _SUFFIX contract. Emit sims_type into the generated m_bias_config, and bump im_mbias mem to 8G since blended catalogues are ~5x grid size and OOM the snakemake 1000M default. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HubFhoU6bH8ij6hcTWxw7s --- src/sp_validation/image_sims.py | 6 +++++- workflow/rules/image_sims.smk | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/image_sims.py b/src/sp_validation/image_sims.py index e259109d..b29ab368 100644 --- a/src/sp_validation/image_sims.py +++ b/src/sp_validation/image_sims.py @@ -152,11 +152,15 @@ def load_catalogs(self, verbose=True): grids_dir = self.cfg["grids_dir"] num = self.cfg["num"] cat_name = self.cfg.get("catalog_name", "shape_catalog_cut_ngmix.fits") + # Grid sims live in ``{branch}_grid_{num}``; other families (e.g. + # blended) in ``{branch}_{num}``. + sims_type = self.cfg.get("sims_type", "grid") + suffix = f"_grid_{num}" if sims_type == "grid" else f"_{num}" # ``sim_names`` (incl. the unsheared reference) comes from the config's # branch map. The +g/-g pool estimator pairs the sheared sims directly; # the reference is loaded for completeness and null-test diagnostics. for name in self.sim_names: - path = f"{grids_dir}/{name}_grid_{num}/{cat_name}" + path = f"{grids_dir}/{name}{suffix}/{cat_name}" if verbose: print(f" Loading {path}") self.cats[name] = _load_cat(path, self.w_cols) diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 4f2c2fed..e8eb09b4 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -379,7 +379,9 @@ rule im_pipeline: n_smp=IMSIM["n_smp"], config_flag=f"-c {PIPELINE_CONFIG_DIR}" if PIPELINE_CONFIG_DIR else "", resources: - mem_mb=16000, + # measured: job uses ~1 core (eff 0.9); post-#843 MaxRSS ~72MB accounting / 2.3GB live-peak; 2cpu/4GB packs ~512 jobs on the usable partition + mem_mb=4000, + cpus_per_task=2, runtime=720, shell: "cd {params.run_dir} && " @@ -464,10 +466,14 @@ rule im_mbias: ), output: results=f"{GRIDS_BASE}/results/m_bias_results.yaml", + resources: + # Blended catalogues are ~5x grid size; the snakemake 1000M default OOMs. + mem_mb=8000, params: cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", grids_base=GRIDS_BASE, num=NUM, + sims_type=SIMS_TYPE, cat_name=f"shape_catalog_cut_{SHAPE}.fits", sif=SIF, sif_pipeline=SIF_PIPELINE, @@ -553,6 +559,7 @@ rule im_mbias: mbias_cfg = { "grids_dir": params.grids_base, "num": params.num, + "sims_type": params.sims_type, "catalog_name": params.cat_name, # Injected shear: from the manifest, the single source of truth. "shear_amplitude": manifest["shear_amplitude"], From c2c0df91dee416197f892c9270de630dd84bee20 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 01:49:37 +0200 Subject: [PATCH 03/36] feat(image_sims): optional exp_num to decouple exposure from tile realization Add an optional `exp_num` config key so the exposure input can be pulled from a different SKiLLS realization than the tiles. Tiles stay on `{branch}_{num}`; exposures resolve `{branch}_{exp_num}` via a new SIM_EXP_BASE mapping, consumed by rule im_init's `sim_exp` symlink source. Unset `exp_num` defaults to `num`, reproducing prior behaviour bit-for-bit. Guarded to non-grid sims (grid names embed sims_type; the pathology this addresses is a blended-family concern). Motivation: the blended-family realizations _1.._4 carry stale cross-linked SP_exp (grid-sims exposures symlinked in, signal-free); _5 is the clean all-real exposure set. This reproduces Fabian's production split (exposures from _5, tiles from _1), unblocking the blended smoke arms whose ngmix flux collapsed ~10x on the contaminated z-branch exposures. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HubFhoU6bH8ij6hcTWxw7s --- workflow/image_sims/config.yaml | 6 ++++++ workflow/rules/image_sims.smk | 22 ++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 144b52b3..35acc694 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -51,6 +51,12 @@ image_sims: # --- simulation grid -------------------------------------------------- sims_type: grid # 'grid' -> *_grid_{num}; anything else -> *_{num} [operational] num: 1 + # Optional: pull exposures from a different realization than tiles (tiles + # from `{branch}_{num}`, exposures from `{branch}_{exp_num}`; unset -> + # exposures follow `num`). Non-grid sims only. Fabian's blended production + # split: tiles from _1, exposures from _5 (the clean all-real exposure set; + # _1.._4 carry signal-free grid-sims SP_exp cross-links). + # exp_num: 5 # Branches this run requests: the unsheared reference plus the four +/- # sheared branches. Injected shear is NOT set here -- it is parsed from each # branch's basic_info.txt by im_manifest. [operational] diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index e8eb09b4..d93300cb 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -92,6 +92,10 @@ _OPERATIONAL_KEYS = { "extract_script", "calibrate_script", } +# Optional keys: recognized but not required; defaults applied in code below. +_OPTIONAL_KEYS = { + "exp_num", +} # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { "sif", @@ -105,7 +109,11 @@ _STRUCTURAL_KEYS = { "tile_ids", } _ALLOWED_KEYS = ( - _SCIENCE_KEYS | _DEPRECATED_KEYS | _OPERATIONAL_KEYS | _STRUCTURAL_KEYS + _SCIENCE_KEYS + | _DEPRECATED_KEYS + | _OPERATIONAL_KEYS + | _OPTIONAL_KEYS + | _STRUCTURAL_KEYS ) _unknown = set(IMSIM) - _ALLOWED_KEYS @@ -156,6 +164,15 @@ SIMS_TYPE = IMSIM["sims_type"] _SUFFIX = f"_{SIMS_TYPE}_{NUM}" if SIMS_TYPE == "grid" else f"_{NUM}" SIM_BASES = list(IMSIM["branches"]) SIMS = [f"{base}{_SUFFIX}" for base in SIM_BASES] +# Optional ``exp_num`` pulls exposures from a different realization than tiles: +# tiles stay on ``{branch}_{num}``, exposures come from ``{branch}_{exp_num}``. +# Non-grid sims only (grid names embed sims_type); see config.yaml for the +# blended use case. +EXP_NUM = IMSIM.get("exp_num", NUM) +if SIMS_TYPE == "grid": + SIM_EXP_BASE = {sim: sim for sim in SIMS} +else: + SIM_EXP_BASE = {f"{base}{_SUFFIX}": f"{base}_{EXP_NUM}" for base in SIM_BASES} MANIFEST = f"{GRIDS_BASE}/manifest.yaml" BUILD_MANIFEST = f"{SPV_REPO}/workflow/scripts/im_build_manifest.py" @@ -340,7 +357,8 @@ rule im_init: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis", sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles", - sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_exp", + # SIM_EXP_BASE maps the tile sim to its exposure-source sim (see above). + sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{SIM_EXP_BASE[wc.sim]}/images/SP_exp", shell: # cfis / input_tiles / input_exp are stable read-only symlinks (used by # get_images, merge, extract); created here but not tracked as outputs, From faa02924678e2072406820763f805c1c8b109636 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:04:00 +0200 Subject: [PATCH 04/36] image_sims: run the ShapePipe processing through ShapePipe's own workflow im_pipeline shelled out per tile to the legacy bash runner (run_job_sp_canfar_v2.0.bash -t image_sims) over example/cfis_image_sims, configs frozen before the real-data chain gained background-RMS vignets, PIXEL_SCALE, chunked ngmix and the new flags. An m measured that way does not calibrate the pipeline that makes the real catalogue. im_shapepipe replaces it: one localrule per shear branch writes a ShapePipe run config (input_type: image_sims, psf_model, tile list, the branch's SP_tiles/SP_exp, run_dir = grids_base/) and runs `sp run`, whose jobs ShapePipe submits through its own profile. The output is a campaign record (ShapePipe commit, resolved image, final_cat count) that im_merge consumes. im_init no longer stages input_tiles/input_exp; its cfis link now points at the checkout's workflow/config/cfis_image_sims, and it writes tiles_.txt, which is both the campaign's tile_list and params.py's tile-ID file. Config: drop config_dir, pipeline_config_dir and n_smp (dead with the bash runner; an A/B of the ngmix ini is now a ShapePipe config change); add sp_profile, sp_jobs, clean_exposures; psf_model defaults to fake, the true PSF of the star-free grid sims. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- workflow/image_sims/config.yaml | 31 ++- workflow/profiles/candide/config.yaml | 7 +- workflow/rules/image_sims.smk | 269 ++++++++++++++++---------- 3 files changed, 196 insertions(+), 111 deletions(-) diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 35acc694..67935f67 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -66,16 +66,27 @@ image_sims: # --- calibration ------------------------------------------------------ shape: ngmix # [operational] - # ShapePipe cfis configs (final_cat.param etc.); default is - # {shapepipe_repo}/example/cfis_image_sims once #766 lands. [operational] - config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims - # ShapePipe config tree for the pipeline stage (run_job's -c override); owns - # the ngmix ini and hence METACAL_PSF. Empty -> run_job's default, - # {shapepipe_repo}/example/cfis_image_sims. Point it at a per-arm copy to - # A/B the metacal PSF without editing the worktree. [operational] - pipeline_config_dir: "" - psf_model: psfex # [operational] - n_smp: -1 # [operational] + + # --- ShapePipe processing (ShapePipe's own Snakemake workflow) --------- + # im_shapepipe runs `{shapepipe_repo}/workflow/bin/sp run` once per branch + # with input_type: image_sims, so shapepipe_repo must be a checkout carrying + # that workflow mode. The config chain is the checkout's + # workflow/config/cfis_image_sims -- change processing there, not here. + # PSF model, as ShapePipe's workflow spells it: `fake` (the true simulation + # PSF; the star-free *_grid_N sims) or `psfex` / `mccd` (sims with stars: + # *_star_grid, non-grid *_N). [operational] + psf_model: fake + # ShapePipe profile (shapepipe_repo/profiles/) that submits the + # campaign's SLURM jobs. [operational] + sp_profile: candide + # Concurrent SLURM jobs per branch campaign (`sp run --jobs`); the branches + # run as concurrent localrules, so the cluster sees up to 5x this. + # [operational] + sp_jobs: 30 + # Rolling exposure-store reclamation (ShapePipe `clean:`): an exposure's + # store (~8 GB) is deleted once every tile reading it has its vignets. + # [operational] + clean_exposures: true # Extract/calibrate scripts run from the sp_validation repo checkout (branch # code, not the baked copies). Point elsewhere for a different checkout. # [operational] diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 0316b909..258145cb 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -59,12 +59,13 @@ executor: slurm # *per node*, and apptainer crashes ("can't start new thread") beyond ~4 # concurrent jobs on a 48-core node. Requesting 12 CPUs/job holds SLURM to # ~4 jobs per 48-core node, under the ceiling. Dropping this to 1 would let -# SLURM pack ~48 jobs onto a node and crash the compute-heavy im_pipeline -# stage (which inherits this default -- it sets mem/runtime but not cpus). +# SLURM pack ~48 jobs onto a node. (The compute-heavy ShapePipe +# processing is not submitted under this profile: im_shapepipe runs +# ShapePipe's own workflow, whose jobs follow ShapePipe's profile.) # # * After launching a real fan-out, VERIFY the request actually landed: # ``squeue -u $USER -o "%C %l"`` must show 12 (CPUs) and the wall clock you -# intended (e.g. 12:00:00 for im_pipeline). A silently-misparsed runtime or +# intended (e.g. 1:00:00 for the default). A silently-misparsed runtime or # cpus shows up here before it wastes a queue slot. default-resources: slurm_account: "cusers" diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index d93300cb..bb2194bf 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -6,6 +6,18 @@ the sp_validation-side half of the split described in per-tile shape catalogues, then sp_validation merges, extracts, calibrates and finally measures the multiplicative/additive shear bias. +One ShapePipe processing definition. The per-tile ShapePipe processing (tile +and exposure stages, up to ``final_cat``) is NOT defined here: ``im_shapepipe`` +drives ShapePipe's own Snakemake workflow (``workflow/bin/sp run`` in +``shapepipe_repo``) once per shear branch, with ``input_type: image_sims``. +The sims therefore pass through the same module order, config chain and +PSF-model switch as the real catalogue -- the overlay config dir +``workflow/config/cfis_image_sims`` differs from the real-data one only in +input naming, the fake-PSF INIs and the merge column list -- so the m measured +here calibrates the pipeline that is actually used. ``psf_model`` is ``fake`` +(the true simulation PSF; star-free ``*_grid_N`` sims) or ``psfex``/``mccd`` +(sims with stars), exactly as ShapePipe's workflow spells it. + Two images, one prefix shape. Architecturally one image could run every stage -- the sp_validation image is built ``FROM`` the ShapePipe image, so it carries both stacks -- but the *published* sp_validation image's environment @@ -18,7 +30,8 @@ binary deps, so until sp_validation is uv-locked with its deps declared (spun off as its own task), each half runs in its own repo's image -- the same split the gate766 baseline ran: -* ShapePipe stages -> ``pipeline`` (raw images -> per-tile cats) and ``merge`` +* ShapePipe stages -> ``shapepipe`` (raw images -> per-tile cats, via ``sp + run``; ``sif_pipeline`` is the run config's ``container:``) and ``merge`` (``create_final_cat`` -> ``final_cat_{sim}.hdf5``) run in ``sif_pipeline`` (the ShapePipe image). * sp_validation stages -> ``manifest``, ``extract`` (-> comprehensive cat), @@ -28,7 +41,8 @@ same split the gate766 baseline ran: Every rule sets ``container: None`` and calls ``apptainer exec`` explicitly through a shared prefix template (``EXEC_PIPELINE`` / ``EXEC`` -- identical env injections, different image), because the images are not the workflow's -top-level container. Everything is parameterised under +top-level container. ``im_shapepipe`` is the exception: ``sp`` owns its jobs' +container through ShapePipe's own profile (``sp_profile``). Everything is parameterised under ``config["image_sims"]`` -- the two ``sif`` keys, repository roots, data roots, the PSF dictionary, the explicit ``tile_ids`` list and the sim/calibration knobs -- so a fresh user drives it from config alone, with no hard-coded clone @@ -85,10 +99,10 @@ _OPERATIONAL_KEYS = { "sims_type", "branches", "shape", - "config_dir", - "pipeline_config_dir", "psf_model", - "n_smp", + "sp_profile", + "sp_jobs", + "clean_exposures", "extract_script", "calibrate_script", } @@ -184,16 +198,21 @@ TILE_IDS = list(IMSIM["tile_ids"]) SHAPE = IMSIM["shape"] MASK_CONFIG = IMSIM["mask_config"] # e.g. config/calibration/mask_v1.X.9_im_sim.yaml PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py" -# ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param). -CONFIG_DIR = IMSIM["config_dir"] -# ShapePipe config dir for the pipeline stage's run_job (``-c`` override; owns -# the ngmix ini and thus METACAL_PSF). Empty -> run_job's default. See -# config.yaml. -PIPELINE_CONFIG_DIR = IMSIM["pipeline_config_dir"] + +# --- ShapePipe processing: ShapePipe's own workflow ------------------------ +# ``sp`` is ShapePipe's workflow launcher; SP_CONFIG_DIR is the config chain it +# selects for ``input_type: image_sims``. The merge and extract steps read that +# dir's ``final_cat.param`` (through the per-sim ``cfis`` link), so the column +# list and the processing that wrote the columns come from one checkout. +SP_LAUNCHER = f"{SHAPEPIPE_REPO}/workflow/bin/sp" +SP_CONFIG_DIR = f"{SHAPEPIPE_REPO}/workflow/config/cfis_image_sims" +PSF_MODEL = IMSIM["psf_model"] +SP_PROFILE = IMSIM["sp_profile"] +SP_JOBS = IMSIM["sp_jobs"] +CLEAN_EXPOSURES = bool(IMSIM["clean_exposures"]) # ShapePipe scripts live in the ShapePipe repo (also baked into its image). CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" -RUN_JOB = f"{SHAPEPIPE_REPO}/scripts/sh/run_job_sp_canfar_v2.0.bash" # Extract/calibrate run from the sp_validation *repo* checkout (bind-mounted), # not the baked copies: the container tracks the branch but lags it, and the # image-sims path needs branch-only fixes (star-catalogue-optional extract, @@ -204,49 +223,76 @@ CALIBRATE = IMSIM["calibrate_script"] COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" # --- container exec prefixes ---------------------------------------------- -# One prefix *shape* for every stage -- two instances, one per image. Three -# env injections make the on-disk branch -# code and the sim PSF win over the image's baked copies: +# One prefix *shape* for every containerised stage (all but im_shapepipe) -- +# two instances, one per image. The env injections make the on-disk branch +# code win over the image's baked copies: # # * PYTHONPATH prepends BOTH repos' ``src`` (ShapePipe first, then # sp_validation), so Python resolves the worktree build before # ``/app``/``/sp_validation`` -- the local-testing counterpart of the # git-ref deps, letting the branch code run without an image rebuild. This -# covers the Python *packages* only: the bash entry points (run_job) and -# the ShapePipe/sp_validation *scripts* are still invoked at the repo paths -# resolved from config (RUN_JOB, CREATE_FINAL_CAT, EXTRACT_INFO, ...), not -# shadowed by PYTHONPATH. -# * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded -# via getexpanded) at this run's PSF dictionary. +# covers the Python *packages* only: the ShapePipe/sp_validation *scripts* +# are still invoked at the repo paths resolved from config +# (CREATE_FINAL_CAT, EXTRACT_INFO, ...), not shadowed by PYTHONPATH. # -# The SLURM env vars are stripped (``env -u ...``) so that when the ShapePipe -# pipeline stage's OpenMPI initialises inside the image it does not try to -# attach to the host SLURM launcher (cf. apptainer_noslurm.sh). The strip is -# harmless for the pure-Python sp_validation stages, so one prefix serves all. +# The SLURM env vars are stripped (``env -u ...``) so that an MPI stack +# initialising inside the image does not try to attach to the host SLURM +# launcher (cf. apptainer_noslurm.sh); harmless for these pure-Python stages. # # ``OMP_NUM_THREADS=1`` is injected here, at the ``apptainer exec`` call, and -# not left to the SLURM profile. The chain is MPI-free: Snakemake fans out one -# job per branch x tile and each job's parallelism is ShapePipe's own internal -# multiprocessing (``-N n_smp``), so the OpenMP/BLAS thread pool inside the -# container must be pinned to 1 to avoid oversubscription. The SLURM profile -# cannot pin it reliably: the slurm executor submits with ``--export=ALL``, -# which propagates the *driver's* ambient environment -- but a Snakemake -# profile only sets CLI flags, never the driver's own env, so an +# not left to the SLURM profile: the slurm executor submits with +# ``--export=ALL``, which propagates the *driver's* ambient environment, and a +# Snakemake profile only sets CLI flags, never the driver's own env, so an # ``OMP_NUM_THREADS`` there would depend on the operator having exported it by -# hand (the implicit, uncommitted state the "one run command" is meant to -# retire). Injecting it on the ``apptainer exec`` line puts it where the -# compute actually runs -- inside the container, independent of the driver's -# env -- the same lever this prefix already uses for PYTHONPATH/PSF_DICT. +# hand. Injecting it on the ``apptainer exec`` line puts it where the compute +# actually runs. (ShapePipe's own profile pins it the same way for the jobs +# ``sp`` submits.) _EXEC_PREFIX = ( "env -u SLURM_JOBID -u SLURM_JOB_ID -u SLURM_PROCID " f"apptainer exec --bind {BINDS} " f"--env PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " - f"--env PSF_DICT={PSF_DICT} --env OMP_NUM_THREADS=1 " + "--env OMP_NUM_THREADS=1 " ) EXEC = _EXEC_PREFIX + SIF # sp_validation stages -EXEC_PIPELINE = _EXEC_PREFIX + SIF_PIPELINE # ShapePipe stages +EXEC_PIPELINE = _EXEC_PREFIX + SIF_PIPELINE # ShapePipe merge stage -JOB_MASK = sum([1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]) + +def sp_run_config(sim): + """ShapePipe workflow run config for one shear branch (one ``sp`` campaign). + + It REPLACES ShapePipe's committed ``workflow/config.yaml`` (``bin/sp``'s + ``SP_RUN_CONFIG``), so every key the campaign needs is stated; processing + knobs this config does not state (``ngmix_chunks``) take the ShapePipe + Snakefile's defaults, never a copy here. ``clean_tiles`` stays off: the + merge reads each tile's make_cat output from the tile store. ``clean`` + (rolling exposure-store reclamation) is ``clean_exposures`` -- a 40-tile + branch touches ~300 exposures at ~8 GB of stores each. + """ + run_dir = f"{GRIDS_BASE}/{sim}" + cfg = { + "input_type": "image_sims", + "psf_model": PSF_MODEL, + "tile_list": f"{run_dir}/tiles_{sim}.txt", + "inputs": { + "tiles": f"{INPUT_SIMS_BASE}/{sim}/images/SP_tiles", + "exposures": f"{INPUT_SIMS_BASE}/{SIM_EXP_BASE[sim]}/images/SP_exp", + }, + "container": SIF_PIPELINE, + "outputs": { + "run_dir": run_dir, + "index_db": f"{run_dir}/index/run_index.sqlite", + }, + "clean": CLEAN_EXPOSURES, + "clean_tiles": False, + "clean_ignore_tiles": [], + } + if PSF_MODEL == "fake": + cfg["psf_dict"] = PSF_DICT + return cfg + + +localrules: + im_shapepipe, wildcard_constraints: @@ -267,13 +313,9 @@ rule im_init_all: expand(f"{GRIDS_BASE}/{{sim}}/params.py", sim=SIMS), -rule im_pipeline_all: +rule im_shapepipe_all: input: - expand( - f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done", - sim=SIMS, - tile=TILE_IDS, - ), + expand(f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", sim=SIMS), rule im_merge_all: @@ -330,18 +372,14 @@ rule im_manifest: rule im_init: - """Stage per-sim run directory: params.py, mask config, ShapePipe configs, - and the raw SKiLLS image inputs. + """Stage per-sim run directory: params.py, mask config, tile list, and the + ``cfis`` link to ShapePipe's image-sims config dir. ``params_im_sim.py`` derives the field name from the directory basename, so the same template serves every sim; ``config_mask.yaml`` and ``cfis`` are symlinks the downstream calibration and merge steps read from cwd. - - ``input_tiles``/``input_exp`` are top-level symlinks to the raw SKiLLS tile - and exposure images; ShapePipe's ``get_images_runner`` resolves them via - ``$SP_DIR/input_{tiles,exp}`` (``$SP_DIR`` is the run dir). ``run_job`` does - not stage these, so ``im_init`` must -- this is what makes ``im_pipeline`` - runnable from raw images, not just from pre-staged intermediates. + ``tiles_{sim}.txt`` is both the ShapePipe campaign's ``tile_list`` and the + tile-ID file ``params.py`` names for the found/missing-tile check. """ input: # Tracked so that editing the params template or mask config re-stages @@ -352,73 +390,108 @@ rule im_init: output: params=f"{GRIDS_BASE}/{{sim}}/params.py", mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", + tiles=f"{GRIDS_BASE}/{{sim}}/tiles_{{sim}}.txt", params: - config_dir=CONFIG_DIR, - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + config_dir=SP_CONFIG_DIR, cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis", - sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles", - # SIM_EXP_BASE maps the tile sim to its exposure-source sim (see above). - sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{SIM_EXP_BASE[wc.sim]}/images/SP_exp", + tile_ids=" ".join(TILE_IDS), shell: - # cfis / input_tiles / input_exp are stable read-only symlinks (used by - # get_images, merge, extract); created here but not tracked as outputs, - # which snakemake will not accept for a symlink/directory. + # cfis is a stable read-only symlink (used by merge and extract); + # created here but not tracked as an output, which snakemake will not + # accept for a symlink to a directory. "mkdir -p $(dirname {output.params}) && " "cp {input.template} {output.params} && " "ln -sf {input.mask_src} {output.mask} && " "ln -sfT {params.config_dir} {params.cfis} && " - "ln -sfT {params.sim_tiles} {params.run_dir}/input_tiles && " - "ln -sfT {params.sim_exp} {params.run_dir}/input_exp" + "printf '%s\\n' {params.tile_ids} > {output.tiles}" + +rule im_shapepipe: + """Run ShapePipe's own workflow over one shear branch (ShapePipe stage). -rule im_pipeline: - """Run ShapePipe on one simulated tile (ShapePipe stage). + Writes the branch's ShapePipe run config (``sp_run_config``) and runs + ``sp run`` on it: one campaign, PREPARE then COMPUTE, whose jobs ``sp`` + submits to SLURM itself through ShapePipe's profile ``sp_profile``. That + is why this is a localrule -- it is a scheduler, not a compute job. The + campaign resumes: rerunning after a failure re-does only incomplete units. - Delegates the module DAG to ShapePipe's own job runner; the sentinel log - marks tile completion for the merge step. This is the compute-heavy, - MPI-bearing stage. + The output is the campaign record: which ShapePipe checkout and image + produced the per-tile catalogues, and how many of the requested tiles have + a ``final_cat``. ``sp run`` exits non-zero if any tile failed (keep-going + lets the rest finish), and so does this rule; read the campaign's + ``index/run_report.json`` for the per-stage verdicts. """ input: - # ``params.py`` is a *tracked* output of ``im_init``, so this one input - # supplies the im_init -> im_pipeline edge. The ``cfis`` symlink the - # shell reads (via {RUN_JOB}) is created by that same im_init shell block - # as an *untracked* side effect -- no rule declares it as an output - # (snakemake will not track a symlink/directory output). Declaring it an - # input here therefore asked the DAG for a file no rule produces: on a - # fresh grids_base it aborted the build with MissingInputException before - # any job ran. It is safe to drop -- cfis exists whenever params does, - # since im_init stages both together. - params=f"{GRIDS_BASE}/{{sim}}/params.py", + tiles=f"{GRIDS_BASE}/{{sim}}/tiles_{{sim}}.txt", output: - done=touch(f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done"), + record=f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", + log: + f"{GRIDS_BASE}/{{sim}}/logs/sp_run.log", params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - psf=IMSIM["psf_model"], - n_smp=IMSIM["n_smp"], - config_flag=f"-c {PIPELINE_CONFIG_DIR}" if PIPELINE_CONFIG_DIR else "", - resources: - # measured: job uses ~1 core (eff 0.9); post-#843 MaxRSS ~72MB accounting / 2.3GB live-peak; 2cpu/4GB packs ~512 jobs on the usable partition - mem_mb=4000, - cpus_per_task=2, - runtime=720, - shell: - "cd {params.run_dir} && " - "{EXEC_PIPELINE} bash {RUN_JOB} " - "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " - "-p {params.psf} -N {params.n_smp} {params.config_flag}" + run_config=lambda wc: sp_run_config(wc.sim), + run_config_path=lambda wc: f"{GRIDS_BASE}/{wc.sim}/shapepipe_run.yaml", + run: + import subprocess + + import yaml + + with open(params.run_config_path, "w") as fh: + fh.write( + "# ShapePipe workflow run config, written by sp_validation's\n" + "# im_shapepipe rule (workflow/rules/image_sims.smk).\n" + ) + yaml.safe_dump(params.run_config, fh, sort_keys=False) + env = ( + f"SP_PROFILE={SP_PROFILE} " + f"SP_RUN_CONFIG={params.run_config_path}" + ) + shell(f"{env} {SP_LAUNCHER} run --jobs {SP_JOBS} > {log} 2>&1") + + def _out(*cmd): + return subprocess.run( + cmd, capture_output=True, text=True, check=True + ).stdout.strip() + + run_dir = params.run_config["outputs"]["run_dir"] + tiles = [t.strip() for t in open(input.tiles) if t.strip()] + final_cats = [ + t for t in tiles + if os.path.exists(f"{run_dir}/tiles/{t[:2]}/{t}/final_cat-{t}.fits") + ] + record = { + "run_config": params.run_config_path, + "sp_profile": SP_PROFILE, + "shapepipe": { + "repo": SHAPEPIPE_REPO, + "branch": _out("git", "-C", SHAPEPIPE_REPO, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _out("git", "-C", SHAPEPIPE_REPO, "rev-parse", "HEAD"), + "dirty": bool(_out("git", "-C", SHAPEPIPE_REPO, "status", "--porcelain")), + }, + # sp's own image resolution (sandbox > cached SIF > ``container:``), + # i.e. the image the campaign's jobs actually ran in. + "image": _out( + "env", f"SP_PROFILE={SP_PROFILE}", + f"SP_RUN_CONFIG={params.run_config_path}", + SP_LAUNCHER, "container", "resolve", + ).splitlines()[-1], + "run_report": f"{run_dir}/index/run_report.json", + "n_tiles": len(tiles), + "n_final_cats": len(final_cats), + } + with open(output.record, "w") as fh: + yaml.safe_dump(record, fh, sort_keys=False) rule im_merge: """Merge per-tile ShapePipe catalogues into final_cat_{sim}.hdf5. ``create_final_cat.py`` lives in the ShapePipe repo/image; run in image_sims - mode (``-I``) it walks the per-tile output under the run directory. + mode (``-I``) it walks the campaign's tile stores under the run directory + (``tiles///output/run_sp_tile_Mc``), selecting the columns in + ShapePipe's image-sims ``final_cat.param``. """ input: - tiles=expand( - f"{GRIDS_BASE}/{{{{sim}}}}/logs/pipeline_{{tile}}.done", - tile=TILE_IDS, - ), + record=f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", output: cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", params: From a4cfb115f7c3c3d17d7ad92cee26a5ed81e41ef2 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:04:37 +0200 Subject: [PATCH 05/36] profiles/candide: set the job cap snakemake requires for the slurm executor Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- workflow/image_sims/Snakefile | 4 ++-- workflow/profiles/candide/config.yaml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile index d7a89065..239809fc 100644 --- a/workflow/image_sims/Snakefile +++ b/workflow/image_sims/Snakefile @@ -17,8 +17,8 @@ command on candide, with the committed SLURM profile owning all scheduling: defaults automatically, so only ``my_run.yaml`` (the science knobs, and any operational override that run wants) is passed on the command line; Snakemake deep-merges the two. The profile supplies the executor, account, partition, -node excludes and job floor -- no ``-j`` needed (the slurm executor sets the -job cap). Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. +node excludes, job floor and job cap (``jobs:``) -- no ``-j`` needed. +Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. The target (``im_mbias``) is given *before* ``--configfile``: Snakemake's ``--configfile`` takes one-or-more paths, so a target placed after it is diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 258145cb..0f8ce491 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -34,6 +34,9 @@ # set none. executor: slurm +# Concurrent SLURM jobs. Required: snakemake refuses remote execution without +# a job cap ("maximum number of parallel jobs ... has to be specified"). +jobs: 50 # Cluster policy applied to every job unless a rule overrides it. The excludes # are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, From f17c7afd93e75f6c9e73478665ff9456d9204c10 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:06:06 +0200 Subject: [PATCH 06/36] profiles/candide: latency-wait 60 s; 5 s misses fresh dirs on the automounts Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- workflow/profiles/candide/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 0f8ce491..56b42739 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -79,8 +79,10 @@ default-resources: # Give an appearing output file a moment on candide's automounted filesystems # before Snakemake calls a job failed for a missing output, and retry a job -# once on transient node failure. -latency-wait: 5 +# once on transient node failure. 5 s was too short for a fresh run dir on +# /n23data1 (outputs a compute node wrote were "missing locally, parent dir not +# present" on the head); 60 s matches ShapePipe's candide profile. +latency-wait: 60 retries: 1 # Keep the SLURM logs of successful jobs (candide debugging), and rerun a job From 67186b36ee78472bc8bd6055e9b92a9a35e4b303 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:06:17 +0200 Subject: [PATCH 07/36] profiles/candide: the exec prefix no longer carries PSF_DICT Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- workflow/profiles/candide/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 56b42739..065991de 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -15,8 +15,8 @@ # # * Container / apptainer settings. The image-sims rules set # ``container: None`` and own their ``apptainer exec`` call through the -# shared ``EXEC`` prefix (one image for every stage, with PYTHONPATH / -# PSF_DICT / OMP_NUM_THREADS injected there). So no +# shared ``EXEC`` prefix (one image per half of the chain, with +# PYTHONPATH / OMP_NUM_THREADS injected there). So no # ``software-deployment-method: apptainer`` / ``apptainer-args`` -- those # would wrap a *second*, redundant container around jobs that already run # inside one. From 6caba1234e62468c541bdb6841965b68dac6b4c3 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:56:17 +0200 Subject: [PATCH 08/36] image_sims: cap concurrent ShapePipe campaigns (disk), 50 jobs each Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- workflow/image_sims/config.yaml | 8 ++++---- workflow/profiles/candide/config.yaml | 6 ++++++ workflow/rules/image_sims.smk | 8 ++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 67935f67..da9c96e8 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -79,10 +79,10 @@ image_sims: # ShapePipe profile (shapepipe_repo/profiles/) that submits the # campaign's SLURM jobs. [operational] sp_profile: candide - # Concurrent SLURM jobs per branch campaign (`sp run --jobs`); the branches - # run as concurrent localrules, so the cluster sees up to 5x this. - # [operational] - sp_jobs: 30 + # Concurrent SLURM jobs per branch campaign (`sp run --jobs`); the profile's + # `sp_campaign` resource caps how many branch campaigns run at once, so the + # cluster sees up to that many times this. [operational] + sp_jobs: 50 # Rolling exposure-store reclamation (ShapePipe `clean:`): an exposure's # store (~8 GB) is deleted once every tile reading it has its vignets. # [operational] diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 065991de..ed9bc729 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -37,6 +37,12 @@ executor: slurm # Concurrent SLURM jobs. Required: snakemake refuses remote execution without # a job cap ("maximum number of parallel jobs ... has to be specified"). jobs: 50 +# Concurrent ShapePipe campaigns (im_shapepipe, one per shear branch; each +# submits its own jobs through ShapePipe's profile). A disk bound -- see the +# rule. Raise it on the command line (--resources sp_campaign=5) when the run +# dir has room for all five branches' exposure stores at once. +resources: + sp_campaign: 2 # Cluster policy applied to every job unless a rule overrides it. The excludes # are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index bb2194bf..758c90fd 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -427,6 +427,14 @@ rule im_shapepipe: record=f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", log: f"{GRIDS_BASE}/{{sim}}/logs/sp_run.log", + resources: + # One unit per concurrent campaign; the profile caps the total + # (``resources: sp_campaign=N``). The cap is a DISK bound: a branch's + # exposure stores (~8 GB each, ~300 per 40-tile branch) are reclaimed + # only once every tile reading them has its vignets, and snakemake + # runs the ready exposure jobs first, so a campaign can hold ~2 TB at + # its peak. + sp_campaign=1, params: run_config=lambda wc: sp_run_config(wc.sim), run_config_path=lambda wc: f"{GRIDS_BASE}/{wc.sim}/shapepipe_run.yaml", From de56f41c3eeac6d31e85ae4be49c7e3765473c26 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 22:53:32 +0200 Subject: [PATCH 09/36] image_sims: append the sp run log, so a retry keeps the failed attempt's Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- workflow/rules/image_sims.smk | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 758c90fd..37beb9e2 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -453,7 +453,12 @@ rule im_shapepipe: f"SP_PROFILE={SP_PROFILE} " f"SP_RUN_CONFIG={params.run_config_path}" ) - shell(f"{env} {SP_LAUNCHER} run --jobs {SP_JOBS} > {log} 2>&1") + # Appended, not truncated: a retry (or a relaunch of the campaign) + # must not erase the log of the attempt that failed. + shell( + f"echo \"=== sp run $(date -Iseconds) ===\" >> {log} && " + f"{env} {SP_LAUNCHER} run --jobs {SP_JOBS} >> {log} 2>&1" + ) def _out(*cmd): return subprocess.run( From 82ea5308046c3af57d1d092fe6b19cdd13e27e9a Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 01:09:03 +0200 Subject: [PATCH 10/36] image_sims: campaign provenance from sp's snapshot; unusable SKiLLS tiles im_shapepipe recorded the ShapePipe commit and dirty state by asking the live checkout when the campaign ENDED -- hours after sp run snapshotted the code it actually ran, so an edit or commit in between put the wrong code in the record. It now copies sp's own launch snapshot (-state/code/snapshot.json: source, branch, head, dirty, taken_at). im_mbias's provenance adds those per-branch records (commit, image, tile and final_cat counts) as shapepipe_campaigns, beside the checkout the merge stage reads; the records are now declared inputs. config.yaml: tile_ids must leave out tiles without data. A tile with no detections fails its ShapePipe campaign and so im_shapepipe; the SKiLLS coadds of grid_1 248.288, grid_2 259.285, grid_3 265.285 and grid_4 281.281/282.281 are all zeros, and 1m2z_2's 257.289 is truncated. The psf_model comment no longer offers *_star_grid (inputs for 1z2z only). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VSU9pPZjpBjzWorAL88aKQ --- workflow/image_sims/config.yaml | 11 +++++++--- workflow/rules/image_sims.smk | 36 +++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index da9c96e8..18d99a77 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -61,7 +61,11 @@ image_sims: # sheared branches. Injected shear is NOT set here -- it is parsed from each # branch's basic_info.txt by im_manifest. [operational] branches: ["1z2z", "1p2z", "1m2z", "1z2p", "1z2m"] - # Tiles to process: an explicit list, the one tile-input mechanism. + # Tiles to process: an explicit list, the one tile-input mechanism. Leave + # out tiles without data: a tile with no detections fails its branch's + # ShapePipe campaign, and so im_shapepipe. In SKiLLS the coadds of grid_1 + # 248.288, grid_2 259.285, grid_3 265.285 and grid_4 281.281/282.281 are + # all zeros, and 1m2z_2's 257.289 is truncated. tile_ids: ["233.293", "237.292", "238.292"] # --- calibration ------------------------------------------------------ @@ -73,8 +77,9 @@ image_sims: # that workflow mode. The config chain is the checkout's # workflow/config/cfis_image_sims -- change processing there, not here. # PSF model, as ShapePipe's workflow spells it: `fake` (the true simulation - # PSF; the star-free *_grid_N sims) or `psfex` / `mccd` (sims with stars: - # *_star_grid, non-grid *_N). [operational] + # PSF; the star-free *_grid_N sims) or `psfex` / `mccd` (the sims with + # stars: non-grid *_N, tiles from N = 1..4 with exp_num: 5; *_star_grid has + # inputs for 1z2z only and gives no m). [operational] psf_model: fake # ShapePipe profile (shapepipe_repo/profiles/) that submits the # campaign's SLURM jobs. [operational] diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 37beb9e2..13d865bc 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -439,6 +439,7 @@ rule im_shapepipe: run_config=lambda wc: sp_run_config(wc.sim), run_config_path=lambda wc: f"{GRIDS_BASE}/{wc.sim}/shapepipe_run.yaml", run: + import json import subprocess import yaml @@ -471,14 +472,20 @@ rule im_shapepipe: t for t in tiles if os.path.exists(f"{run_dir}/tiles/{t[:2]}/{t}/final_cat-{t}.fits") ] + # The code the campaign ran is sp's launch snapshot, not whatever the + # checkout holds when the campaign ends (hours later, possibly another + # commit): sp records the snapshot's commit and dirty state itself. + with open(f"{run_dir}-state/code/snapshot.json") as fh: + snapshot = json.load(fh) record = { "run_config": params.run_config_path, "sp_profile": SP_PROFILE, "shapepipe": { - "repo": SHAPEPIPE_REPO, - "branch": _out("git", "-C", SHAPEPIPE_REPO, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _out("git", "-C", SHAPEPIPE_REPO, "rev-parse", "HEAD"), - "dirty": bool(_out("git", "-C", SHAPEPIPE_REPO, "status", "--porcelain")), + "repo": snapshot["source"], + "branch": snapshot["branch"], + "commit": snapshot["head"], + "dirty": snapshot["dirty"], + "snapshot_taken_at": snapshot["taken_at"], }, # sp's own image resolution (sandbox > cached SIF > ``container:``), # i.e. the image the campaign's jobs actually ran in. @@ -568,6 +575,10 @@ rule im_mbias: cats=expand( f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS ), + # The per-branch ShapePipe campaign records, for provenance. + campaigns=expand( + f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", sim=SIMS + ), output: results=f"{GRIDS_BASE}/results/m_bias_results.yaml", resources: @@ -629,6 +640,17 @@ rule im_mbias: ) return m.group(1).decode() if m else None + def _campaign(path): + """The provenance half of im_shapepipe's campaign record.""" + with open(path) as fh: + record = yaml.safe_load(fh) + return { + **record["shapepipe"], + "image": record["image"], + "n_tiles": record["n_tiles"], + "n_final_cats": record["n_final_cats"], + } + # Manifest hash: sha256 of the exact bytes im_manifest wrote, so the # result records which injected-shear facts it was computed against. with open(input.manifest, "rb") as fh: @@ -640,10 +662,16 @@ rule im_mbias: "branch": _git(params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD"), "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), }, + # The checkout the merge stage runs create_final_cat from. "shapepipe": { "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), }, + # The code and image each branch's tiles were processed with (sp's + # launch snapshot, via im_shapepipe's campaign record). + "shapepipe_campaigns": { + sim: _campaign(path) for sim, path in zip(SIMS, input.campaigns) + }, "containers": { "sif": params.sif, "ghcr_revision": _sif_revision(params.sif), From a1d004ab5f728232bfb63aba71a7fc8ece92c5e6 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 10:21:34 +0200 Subject: [PATCH 11/36] image_sims: tile_store_root passes ShapePipe's tile-store location per run null (the default) keeps the profile's node-local store and leaves the key out of the branch run configs, so earlier campaigns keep their params. The star sims set it to NFS: candide's 31 GB node /tmp cannot hold their ~5 GB tile stores. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Qv5MwRiVKa8wnH665dWBia --- workflow/image_sims/config.yaml | 7 +++++++ workflow/rules/image_sims.smk | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 18d99a77..4144a8ef 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -92,6 +92,13 @@ image_sims: # store (~8 GB) is deleted once every tile reading it has its vignets. # [operational] clean_exposures: true + # Where each tile's ShapePipe vignette store lives (ShapePipe run-config + # `tile_store_root`, bound to /local/scratch by `sp run`). null keeps the + # profile's node-local store -- on candide the node's 31 GB /tmp, which + # dense star-sim tiles (~5 GB each) and failed groups' orphans fill, failing + # every later tile group on that node. Set a shared directory for the star + # sims (all branches and runs can share one root). [operational] + tile_store_root: null # Extract/calibrate scripts run from the sp_validation repo checkout (branch # code, not the baked copies). Point elsewhere for a different checkout. # [operational] diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 13d865bc..ec5f44b4 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -103,6 +103,7 @@ _OPERATIONAL_KEYS = { "sp_profile", "sp_jobs", "clean_exposures", + "tile_store_root", "extract_script", "calibrate_script", } @@ -210,6 +211,11 @@ PSF_MODEL = IMSIM["psf_model"] SP_PROFILE = IMSIM["sp_profile"] SP_JOBS = IMSIM["sp_jobs"] CLEAN_EXPOSURES = bool(IMSIM["clean_exposures"]) +# Where each tile_shape group's vignette store lives (ShapePipe's +# ``tile_store_root``; ``sp run`` binds it to /local/scratch). None keeps the +# profile's node-local bind -- and keeps the key out of the run config, so +# campaigns launched before it existed see the same params. +TILE_STORE_ROOT = IMSIM["tile_store_root"] # ShapePipe scripts live in the ShapePipe repo (also baked into its image). CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" @@ -288,6 +294,8 @@ def sp_run_config(sim): } if PSF_MODEL == "fake": cfg["psf_dict"] = PSF_DICT + if TILE_STORE_ROOT: + cfg["tile_store_root"] = TILE_STORE_ROOT return cfg From fe62876af184e8b5cc5c1796aec2a3785180394f Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 18:47:52 +0200 Subject: [PATCH 12/36] image_sims: native shapepipe_run-per-unit prototype, adapted from Cail's real-data workflow A from-scratch, one-tile-scale port of the image-sims chain onto the per-unit manifest/completeness pattern from shapepipe/workflow/ (calling shapepipe_run directly, no run_job_sp_canfar_v2.0.bash), built and verified step by step (prepare -> exposure -> tile chain -> tile_ngmix -> tile_make_cat) against the 240.291 reference tile from grid_1, plus clean_exposure/clean_tile reclamation reusing shapepipe's own scripts unmodified. Kept separate from feat/imsims-shapepipe-workflow (the more mature, overlay-config-based port already in progress there) -- this is an independent, simpler exploration, not meant to replace it. --- workflow/image_sims/native/Snakefile | 419 ++++++++++++++++++ workflow/image_sims/native/config.yaml | 22 + .../config_exp_Gie_symlink.ini | 91 ++++ .../config/cfis_image_sims/config_exp_Sp.ini | 77 ++++ .../cfis_image_sims/config_exp_psfex.ini | 66 +++ .../config/cfis_image_sims/config_tile_Fe.ini | 72 +++ .../config_tile_Git_symlink.ini | 94 ++++ .../cfis_image_sims/config_tile_Mc_psfex.ini | 88 ++++ .../cfis_image_sims/config_tile_Mh_exp.ini | 81 ++++ .../config_tile_Ng_batch_psfex_sx.ini | 95 ++++ .../config_tile_PiViVi_canfar_sx.ini | 167 +++++++ .../config/cfis_image_sims/config_tile_Sx.ini | 123 +++++ .../config/cfis_image_sims/default.conv | 5 + .../cfis_image_sims/default_noimaflags.param | 65 +++ .../config/cfis_image_sims/default_tile.sex | 133 ++++++ workflow/image_sims/native/rules/exposure.smk | 69 +++ workflow/image_sims/native/rules/prepare.smk | 94 ++++ workflow/image_sims/native/rules/tile.smk | 190 ++++++++ workflow/image_sims/native/run.example.yaml | 44 ++ workflow/image_sims/native/run.sh | 138 ++++++ .../native/scripts/build_forest_sims.py | 46 ++ .../native/scripts/completeness_sims.py | 268 +++++++++++ .../image_sims/native/scripts/status_sims.py | 145 ++++++ workflow/image_sims/native/tiles.example.txt | 1 + 24 files changed, 2593 insertions(+) create mode 100644 workflow/image_sims/native/Snakefile create mode 100644 workflow/image_sims/native/config.yaml create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_exp_Gie_symlink.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_exp_Sp.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_exp_psfex.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_Fe.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_Git_symlink.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_Mc_psfex.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_Mh_exp.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/config_tile_Sx.ini create mode 100644 workflow/image_sims/native/config/cfis_image_sims/default.conv create mode 100644 workflow/image_sims/native/config/cfis_image_sims/default_noimaflags.param create mode 100644 workflow/image_sims/native/config/cfis_image_sims/default_tile.sex create mode 100644 workflow/image_sims/native/rules/exposure.smk create mode 100644 workflow/image_sims/native/rules/prepare.smk create mode 100644 workflow/image_sims/native/rules/tile.smk create mode 100644 workflow/image_sims/native/run.example.yaml create mode 100755 workflow/image_sims/native/run.sh create mode 100644 workflow/image_sims/native/scripts/build_forest_sims.py create mode 100644 workflow/image_sims/native/scripts/completeness_sims.py create mode 100644 workflow/image_sims/native/scripts/status_sims.py create mode 100644 workflow/image_sims/native/tiles.example.txt diff --git a/workflow/image_sims/native/Snakefile b/workflow/image_sims/native/Snakefile new file mode 100644 index 00000000..98820d1e --- /dev/null +++ b/workflow/image_sims/native/Snakefile @@ -0,0 +1,419 @@ +"""Image-sims orchestration, ported onto Cail's native shapepipe_run-per-unit +design (shapepipe/workflow/{Snakefile,rules/*.smk}, CosmoStat/shapepipe#848). + +Being built up step by step, one stage at a time, against ONE reference tile +(240.291, branch 1z2z_grid_1) before broadening to the full tile list and the +five sheared branches. Currently covers, end to end: + PREPARE: tile_get_images -> tile_uncompress -> tile_find_exposures + COMPUTE, exposure chain: exp_get_images -> exp_split (no exp_psf) + COMPUTE, tile chain: tile_exp_forest, tile_merge_headers, tile_detect, + then {tile_fake_psf, tile_vignets} (parallel, both off tile_detect), + tile_ngmix, tile_make_cat (no tile_merge_cats: ngmix is one call, not + N chunks -- see config_tile_Ng_batch_psfex_sx.ini) + +Like the real design, this is now TWO snakemake invocations, gated by +SP_PHASE (prepare/compute) -- see below for why: tile_find_exposures' +output is what tells us which exposures exist, so a single static parse +can't declare exposure-level rules before that stage has actually run. + +Adaptations from the real-data design, and why (see the port plan for the +full investigation): + * Retrieval is symlink, not VOSpace (`-r symlink` in the Gen-2 dispatch); + the committed configs already reflect this (config_tile_Git_symlink.ini). + * tile_uncompress is FAKED: image-sims weights are already uncompressed, so + this rule symlinks the weight into a synthetic + run_sp_tile_Uz/uncompress_fits_runner/output/ dir instead of calling + shapepipe_run -- but still goes through the same manifest/completeness + contract as every other stage, so downstream rules can't tell the + difference (mirrors run_job_sp_canfar_v2.0.bash's job-2 special case). + * tile_numbers.txt is DASH form ("240-291"), not dot form -- image-sims + configs' INPUT_NUMBERING expects it (verified against a completed + production tile); real-data's unit_pre writes dot form. + * $SP_DIR (the sim root, holding input_tiles/input_exp/cfis) is exported + alongside $SP_RUN -- the image-sims configs read $SP_DIR/input_tiles + directly, unlike real data's $SP_INPUT_TILES indirection. + * Every forked ini under config/cfis_image_sims/ sets RUN_DATETIME=False: + the upstream example/cfis_image_sims/*.ini mostly default to True + (timestamped run dirs), which the manifest design's fixed-path lookup + and unit_pre's `rm -rf $SP_RUN/output/` idempotency both require + to be False. + + * The index is plain JSON, not sqlite (Cail's build_index.py writes + sqlite because DR6 has hundreds of concurrent job re-parses racing to + read/write it -- see build_index.py's docstring). At one-tile-to-one-grid + scale under local/`--cores` execution there is one writer, so a JSON + file under `workflow.is_main_process` is the same guarantee with far + less machinery. Revisit if/when this moves to the SLURM executor with + per-job re-parses. + * No EXP name-mapping table: real data needs one because an exposure's + base id ("2605805") differs from its filename ("2605805p"). Image-sims + exposure ids ARE the filename base (confirmed: Gen-2's own dispatch + comment "2182795p (data) or 208659 (image_sims)"), so `exp_name` is + just the wildcard itself -- no index lookup needed for it. + + * tile_fake_psf has no real-data counterpart at all: image sims use a + known, injected PSF (fake_psf_runner, reading the tile's own sexcat), + not a fitted PSFEx model, so it runs at the TILE level as a sibling of + tile_vignets rather than folded into an exposure-level exp_psf. + * No tile_merge_cats: the real design concatenates N ngmix chunk + catalogues before make_cat reads them; image sims runs ngmix as ONE + call over the whole tile (Gen-2 never chunked it), so tile_make_cat + reads tile_ngmix's output directly. + * PRODUCTS_DIR defaults to RUN_DIR (one root) -- the real design's + scratch/persistent split (D5) is a reclaim-at-DR6-scale concern that + doesn't apply yet at one-tile-to-one-grid scale. + +clean_exposure/clean_tile reuse shapepipe's own workflow/scripts/{clean_exposure,clean_tile}.py +UNMODIFIED (stdlib-only, pure CLI) -- see rules/exposure.smk and rules/tile.smk. +The one addition their real-data equivalents needed: tile_exp_split's +ancient()-and-drop cut for a FINISHED tile's reclaimed exposure edges (see +tile_exp_split's docstring), and `--rerun-triggers` must exclude `input` for +that cut to actually suppress reruns (nibi's profile does this; run.sh does +too) -- otherwise a later invocation that appends tiles sees a since-cleaned +exposure's vanished manifest as "input files updated" and rebuilds it, +cascading across every tile that shares it (exposures ARE shared across +neighbouring tiles here, same as real data -- verified on grid_1). + +Not yet ported (later steps): ngmix chunking, node-local staging + group +fusion -- Phase-3 concerns once this chain is verified correct end to end +against the Gen-2 reference tile. +""" + +import json +import os +from pathlib import Path + +from snakemake.exceptions import WorkflowError + +# config.yaml carries only OPERATIONAL defaults (safe to share across users +# and runs, e.g. psf_model) -- see its own header comment. Everything that +# differs from run to run or user to user (tile_list, sim, inputs, the run +# dir, the container) has NO default here and must come from a run config +# layered on top: +# +# snakemake -s Snakefile --configfile config.yaml --configfile my_run.yaml +# +# (run.sh does this for you via -c/--config; see run.example.yaml for the +# template.) Later --configfile args override earlier ones, so a run config +# never needs to repeat the operational defaults. +configfile: str(Path(workflow.snakefile).parent / "config.yaml") + +_REQUIRED = ("tile_list", "sim", "inputs", "outputs", "container", "psf_dict", + "shapepipe_repo") +_missing = [k for k in _REQUIRED if k not in config] +if _missing: + raise WorkflowError( + f"Missing required run-config key(s) {_missing} -- these have no " + "default in config.yaml and must be set in a run config passed via " + "an extra --configfile (see run.example.yaml, and run.sh -c/--config)." + ) +for _k in ("tiles", "exposures"): + if _k not in config["inputs"]: + raise WorkflowError(f"run config: inputs.{_k} is required") +if "run_dir" not in config["outputs"]: + raise WorkflowError("run config: outputs.run_dir is required") + +container: config["container"] + +RUN_DIR = Path(config["outputs"]["run_dir"]) +# One root for now -- see the module docstring's PRODUCTS_DIR note. +PRODUCTS_DIR = Path(config["outputs"].get("products_dir") or RUN_DIR) +# Repo-relative, not a config key: the ini chain versions with the rules that +# set the env vars they interpolate (same reasoning as the real design's "no +# config_src knob" -- see shapepipe/workflow/README.md). +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis_image_sims" +SCRIPTS = Path(workflow.basedir) / "scripts" +INPUTS = config["inputs"] +PSF_DICT = config["psf_dict"] +# clean_exposure.py / clean_tile.py are reused UNMODIFIED from shapepipe's own +# workflow (stdlib-only, pure CLI, no dependency on its Snakefile globals or +# on sqlite -- verified by reading them). No fork needed, unlike +# completeness.py/build_forest.py. +SHAPEPIPE_SCRIPTS = Path(config["shapepipe_repo"]) / "workflow" / "scripts" + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- sim-root setup (mirrors sp_validation's existing im_init rule) -------- +# Static, tile-independent, idempotent -- done once at parse time rather than +# as a rule, same as the real Snakefile's own mkdir-at-parse-time bits. +RUN_DIR.mkdir(parents=True, exist_ok=True) +for name, target in (("input_tiles", INPUTS["tiles"]), + ("input_exp", INPUTS["exposures"]), + ("cfis", CONFIG_DIR)): + link = RUN_DIR / name + if not link.is_symlink() or link.resolve() != Path(target).resolve(): + if link.exists() or link.is_symlink(): + link.unlink() + link.symlink_to(target, target_is_directory=True) + +import sys +sys.path.insert(0, str(SCRIPTS)) +from completeness_sims import STAGE_DIR # noqa: E402 + +# SP_PHASE gates the index build the same way it does in the real design +# (shapepipe/workflow/Snakefile): 'prepare' or 'compute', set by the caller +# (see this repo's README once written; for now: `SP_PHASE=prepare snakemake +# prepare_all_tiles`, then `SP_PHASE=compute snakemake all`). +PHASE = os.environ.get("SP_PHASE", "") +if PHASE not in ("prepare", "compute"): + raise WorkflowError( + f"SP_PHASE={PHASE!r} must be 'prepare' or 'compute' -- this workflow " + "is two invocations, same as the real-data design (see the " + "Snakefile's module docstring).") + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", + +# --- sharded stores (kept even at one-tile scale, for parity with the +# real design and so scaling up later is a config change, not a rewrite) --- +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") +PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") + +def tile_dir(tile): + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" + +def exp_dir(exp): + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def final_cat(tile): + return f"{PRODUCTS_DIR}/tiles/{tile[:2]}/{tile}/final_cat-{tile}.fits" + +def tile_dash(tile): + return tile.replace(".", "-") + +def tile_finished(tile): + """True once this tile has published its final catalogue. + + The tile-finished marker AND the cut-the-reclaimed-edges trigger (see + tile_exp_split below) -- same as the real design's tile_finished(), and + for the same reason: a finished tile has already extracted everything + it will ever read from its exposures, so it has no business rebuilding + them just because clean_exposure deleted their manifests. + """ + return Path(final_cat(tile)).exists() + + +def tile_exp_split(wc): + """This tile's exposures' exp_split manifests -- the DAG edge for every + tile-level rule that reads exposure products (through the forest). + + For a FINISHED tile, manifests clean_exposure has already reclaimed are + DROPPED from this list rather than declared (and every surviving one is + wrapped `ancient()`) -- otherwise a later `sp run`-equivalent invocation + that appends tiles would see "my input vanished" on every tile sharing a + since-cleaned exposure and rebuild it from scratch, cascading across + every spatial neighbour (measured on the real design's own fixture: one + damaged tile reached all four in a shared-exposure quad without this + cut). See exposure.smk's clean_exposure rule for the write side, and the + module docstring on why this also needs `input` out of + --rerun-triggers. + """ + paths = [exp_manifest(e, "exp_split") for e in tile_exposures(wc.tile)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] + +# --- parse-time index build + load ------------------------------------ +INDEX_JSON = RUN_DIR / "index.json" + +def _build_index(): + """Read each tile's tile_find_exposures output, write {tile: [exp_ids]}. + + Merges with whatever is already on disk rather than overwriting, so the + index ACCUMULATES across invocations the same way the real design's does + (appending a tile later does not forget earlier tiles that are not in + THIS invocation's tile_list). + """ + index = json.loads(INDEX_JSON.read_text()) if INDEX_JSON.exists() else {} + for t in TILES: + fe_dir = Path(tile_dir(t)) / "output" / "run_sp_tile_Fe" / \ + "find_exposures_runner" / "output" + matches = sorted(fe_dir.glob("exp_numbers-*.txt")) if fe_dir.is_dir() else [] + if not matches: + continue + index[t] = [ln.strip() for ln in matches[0].read_text().splitlines() + if ln.strip()] + INDEX_JSON.parent.mkdir(parents=True, exist_ok=True) + INDEX_JSON.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n") + +if PHASE == "compute" and workflow.is_main_process: + _build_index() + +TILE_EXP = json.loads(INDEX_JSON.read_text()) if INDEX_JSON.exists() else {} + +def tile_exposures(tile): + return TILE_EXP.get(tile, []) + +TILES_READY = [t for t in TILES if tile_exposures(t)] + +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} " + f"ready (index {INDEX_JSON}). Run the prepare phase first: " + "SP_PHASE=prepare snakemake prepare_all_tiles") + +# --- reclamation (D5-equivalent; see the real design's Snakefile for the +# full rationale, which is unchanged here) ---------------------------------- +# Off by default (config.yaml); a run config opts in explicitly. +CLEAN = bool(config.get("clean", False)) and PHASE == "compute" +CLEAN_TILES = bool(config.get("clean_tiles", False)) and PHASE == "compute" +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) + +# EXP_TILES: exposure -> every tile in the (accumulating) index that reads +# it -- the inverse of TILE_EXP. Exposures genuinely are shared across +# neighbouring tiles here too (verified: exposure 2086271 covers both +# 240.291 and 239.291 in grid_1), so this is not a real-data-only concern. +EXP_TILES = {} +for _t, _exps in TILE_EXP.items(): + for _e in _exps: + EXP_TILES.setdefault(_e, []).append(_t) + +def tombstone(exp): + return f"{exp_dir(exp)}/cleaned.json" + +def tile_tombstone(tile): + return f"{tile_dir(tile)}/cleaned.json" + +def exp_consumers(exp): + return EXP_TILES.get(exp, []) + +def clean_consumers(exp): + return sorted(t for t in exp_consumers(exp) if t not in CLEAN_IGNORE_TILES) + +def clean_targets(): + """Exposures eligible for reclamation THIS invocation: every consuming + tile is either out of this run's scope (its vignets can't build here) or + already has its tile_vignets manifest on disk. See exposure.smk's + clean_exposure rule -- this feeds `rule all`, not the clean rule's own + `input:` (which stays lambda wc, per-exposure, for the ordering edge).""" + if not CLEAN: + return [] + out = [] + for exp, tiles in EXP_TILES.items(): + cons = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() + for t in cons): + out.append(tombstone(exp)) + return sorted(out) + +def clean_tile_targets(): + """In-scope finished tiles eligible for whole-store reclamation. No + eligibility test needed (a tile has no consumer outside itself).""" + if not CLEAN_TILES: + return [] + return sorted(tile_tombstone(t) for t in TILES_READY) + +READY_SET = set(TILES_READY) + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1))) + + +def unit_pre(stage, unit, *, exp_name=None, forest=None, pre_run=()): + """The image-sims unit-furniture + environment prologue, as bash. + + See shapepipe/workflow/Snakefile's unit_pre for the real-data original; + the differences are in this file's module docstring. `unit` is a tile ID + for a tile-level stage, an exposure base id for an exposure-level one -- + STAGE_DIR says which. `forest`, when given, exports $SP_EXP (the + per-tile exposure symlink view configs scan for multi-epoch products). + """ + level, subdir = STAGE_DIR[stage] + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_DIR='{RUN_DIR}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + if level == "tile": + lines.append( + f"printf '%s\\n' '{tile_dash(unit)}' > \"$SP_RUN/tile_numbers.txt\"") + else: + # The pseudo-Fe file exp_get_images' config reads as its + # find_exposures_runner input -- same mechanism (and same path) as + # the real design's unit_pre, and as Gen-2's init_exp_work_dir. + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + lines += list(pre_run) + lines += [ + f'rm -rf "$SP_RUN/output/{subdir}"', + # ShapePipe's own run-history log ($SP_RUN/output/log_run_sp.txt) + # gets one line appended per shapepipe_run call and is NEVER + # cleared by the rm -rf above (it lives one level up, shared by + # every stage in this $SP_RUN). A rerun of this stage -- a retry, + # a resume, iterating during development -- then leaves TWO + # entries for run_sp_tile_, and any OTHER stage whose config + # references this one by exact name (not `last:`) fails with + # "More than one run found matching search string" even though + # only one output directory exists. Prune this stage's own lines + # before every run so reruns can never accumulate a duplicate. + f'[ -f "$SP_RUN/output/log_run_sp.txt" ] && ' + f'grep -vF "$SP_RUN/output/{subdir} " "$SP_RUN/output/log_run_sp.txt" ' + f'> "$SP_RUN/output/.log_run_sp.tmp" || true', + '[ -f "$SP_RUN/output/.log_run_sp.tmp" ] && ' + 'mv "$SP_RUN/output/.log_run_sp.tmp" "$SP_RUN/output/log_run_sp.txt" || true', + 'cd "$SP_RUN"', + ] + return "\n".join(lines) + + +def sp_shell(stage, config_name, *, post=""): + """The rule's shell string: prologue, one shapepipe_run, one completeness + check. See shapepipe/workflow/Snakefile's sp_shell for the pattern this + mirrors. `post` is extra bash between the check and `exit $rc` + (tile_make_cat uses it to publish final_cat, guarded on rc).""" + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" || rc=$?\n" + f"python {SCRIPTS}/completeness_sims.py check {stage} {{output.manifest}}" + " --log {log} --job-rc \"$rc\" || rc=1\n" + f"{post}" + "exit $rc\n" + ) + + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" + +localrules: all, prepare_all_tiles, clean_exposure, clean_tile + +# Invocation 2 (SP_PHASE=compute): everything that needs the index. Declared +# FIRST so it is snakemake's implicit default target -- matching the +# real-data Snakefile's ordering, and why: a bare `snakemake` with no target +# under SP_PHASE=compute must build the compute DAG, not silently re-target +# prepare_all_tiles because it happened to be declared first. +rule all: + input: + [final_cat(t) for t in TILES_READY], + clean_targets(), + clean_tile_targets(), + +# Invocation 1 (SP_PHASE=prepare): the static per-tile DAG. Always named +# explicitly on the command line (`snakemake ... prepare_all_tiles`), never +# relied on as a default target. +rule prepare_all_tiles: + input: + [tile_manifest(t, "tile_find_exposures") for t in TILES] diff --git a/workflow/image_sims/native/config.yaml b/workflow/image_sims/native/config.yaml new file mode 100644 index 00000000..ea555ce3 --- /dev/null +++ b/workflow/image_sims/native/config.yaml @@ -0,0 +1,22 @@ +# OPERATIONAL defaults for the native image-sims workflow -- committed, +# shared across users and runs. The .smk files read these bare (never with a +# literal fallback), so this file is their one home. +# +# Everything that changes from run to run or user to user (tile_list, sim, +# inputs.tiles/inputs.exposures, outputs.run_dir, container) has NO default +# here: it must come from a run config layered on top via a second +# --configfile (later ones override earlier ones). See run.example.yaml for +# the template, and run.sh's -c/--config flag. + +psf_model: psfex + +# Reclamation (off by default -- a run config opts in explicitly, same +# convention as the real design's clean:/clean_tiles: keys). clean: +# reclaims an exposure's store once every consuming tile has its vignets; +# clean_tiles reclaims a finished tile's whole store once its final_cat +# exists. See the Snakefile's module docstring for the cascade-avoidance +# mechanism this depends on (tile_exp_split's ancient()-and-drop cut, and +# run.sh's --rerun-triggers excluding `input`). +clean: false +clean_tiles: false +clean_ignore_tiles: [] diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_exp_Gie_symlink.ini b/workflow/image_sims/native/config/cfis_image_sims/config_exp_Gie_symlink.ini new file mode 100644 index 00000000..8e98851f --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_exp_Gie_symlink.ini @@ -0,0 +1,91 @@ +# ShapePipe configuration file for: get exposure images +# +# Forked from shapepipe/example/cfis_image_sims/config_exp_Gie_symlink.ini +# for the native workflow: RUN_DATETIME -> False (see +# config_tile_Git_symlink.ini's header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_MODULE = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_DIR/input_exp/,$SP_DIR/input_exp/,$SP_DIR/input_exp/ + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = simu_image-0000000,simu_weight-0000000,simu_flag-0000000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits,.fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{7} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-,weight-,flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_exp_Sp.ini b/workflow/image_sims/native/config/cfis_image_sims/config_exp_Sp.ini new file mode 100644 index 00000000..12e9c2e1 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_exp_Sp.ini @@ -0,0 +1,77 @@ +# ShapePipe configuration file for single-exposures, split images +# +# Forked from shapepipe/example/cfis_image_sims/config_exp_Sp.ini for the +# native workflow: RUN_DATETIME -> False (see config_tile_Git_symlink.ini's +# header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = last:get_images_runner + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_exp_psfex.ini b/workflow/image_sims/native/config/cfis_image_sims/config_exp_psfex.ini new file mode 100644 index 00000000..7de37849 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_exp_psfex.ini @@ -0,0 +1,66 @@ +# ShapePipe configuration file for fake PSF postage stamps (image simulations) +# +# NOTE the filename: kept as config_exp_psfex.ini to match Gen-2's own +# convention (job 64's config slot is reused for fake_psf at the tile +# level), but MODULE is fake_psf_runner, not psfex -- this runs with +# SP_RUN = the TILE work dir, reading the tile's own sexcat (job 256 / +# tile_detect), not any exposure product. See tile.smk's tile_fake_psf rule. +# +# Forked from shapepipe/example/cfis_image_sims/config_exp_psfex.ini for the +# native workflow: RUN_DATETIME -> False (see config_tile_Git_symlink.ini's +# header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +VERBOSE = True + +RUN_NAME = run_sp_tile_fpsf + +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +MODULE = fake_psf_runner + +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +LOG_NAME = log_sp + +RUN_LOG_NAME = log_run_sp + +INPUT_DIR = $SP_RUN/output + +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +SMP_BATCH_SIZE = 1 + +TIMEOUT = 96:00:00 + + +## Module options +[FAKE_PSF_RUNNER] + +INPUT_MODULE = sextractor_runner + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +NUMBERING_SCHEME = -000-000 + +# Path to the pickled PSF dictionary. The value is exported by the +# orchestrating workflow (its psf_dict config key), expanded here from the +# environment; on candide it points at the SKiLLS PSF dictionary. +PSF_DICT_PATH = $PSF_DICT diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_Fe.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Fe.ini new file mode 100644 index 00000000..3570e0e9 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Fe.ini @@ -0,0 +1,72 @@ +# ShapePipe configuration file for: find exposures +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_Fe.ini for the +# native workflow: RUN_DATETIME -> False (see config_tile_Git_symlink.ini's +# header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = last:get_images_runner + +FILE_PATTERN = CFIS_simu_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +COLNUM = 2 + +EXP_PREFIX = simu_image- diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_Git_symlink.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Git_symlink.ini new file mode 100644 index 00000000..33b21b03 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Git_symlink.ini @@ -0,0 +1,94 @@ +# ShapePipe configuration file for: get tile images +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_Git_symlink.ini +# for the native (shapepipe_run-per-unit) image-sims workflow: RUN_DATETIME +# flipped to False so the output dir has a FIXED name (run_sp_tile_Git, no +# timestamp suffix) -- required by the manifest/completeness design, which +# looks up stage dirs by a literal path, and by unit_pre's `rm -rf +# $SP_RUN/output/` idempotency guarantee at the start of every run. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_DIR/input_tiles, $SP_DIR/input_tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS_simu_image-000-000, CFIS_simu_weight-000-000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}-\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_simu_image-, CFIS_simu_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# Copy command options, optional +RETRIEVE_OPTIONS = -L diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_Mc_psfex.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Mc_psfex.ini new file mode 100644 index 00000000..bfd10b8b --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Mc_psfex.ini @@ -0,0 +1,88 @@ +# ShapePipe post-run configuration file: create final catalogs for psfex +# +# No merge_cats step: unlike the real-data design (which concatenates N +# ngmix chunk catalogues before make_cat reads them), image sims has one +# ngmix output directly (no chunking -- see config_tile_Ng_batch_psfex_sx.ini's +# header comment), so make_cat reads it straight from run_sp_tile_Ng. +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_Mc_psfex.ini for +# the native workflow: RUN_DATETIME -> False, explicit (see +# config_tile_Git_symlink.ini's header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mc_psfex + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:fake_psf_runner, last:ngmix_runner + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +# Star/galaxy spread-model classification. The image-sims pipeline has no +# spread_model_runner stage (make_cat inputs are sextractor + fake_psf + +# ngmix only), so there is nothing to classify -- disable it. The newer +# make_cat_runner reads this key unconditionally (NoOptionError if absent). +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_Mh_exp.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Mh_exp.ini new file mode 100644 index 00000000..fdc0d12c --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Mh_exp.ini @@ -0,0 +1,81 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_Mh_exp.ini for +# the native workflow: RUN_DATETIME -> False (see +# config_tile_Git_symlink.ini's header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP + +# If True, log a warning and continue when split_exp_runner output is missing +# for some exposures (e.g. those that failed job 16). If False (default), +# raise an error and stop. +WARN_MISSING_EXP = False diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini new file mode 100644 index 00000000..7d4a4688 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini @@ -0,0 +1,95 @@ +# ShapePipe configuration file for tiles: ngmix +# +# No chunking: unlike the real-data design's tile_ngmix (8-way scatter for +# DR6-scale I/O reasons), Gen-2's image-sims dispatch always ran ngmix as +# ONE call over the whole tile (job 1024 = ngmix_runner:1, no split). Kept +# that way here; chunking is a later-phase decision if image-sims tiles +# turn out to need it (they are far smaller than DR6 tiles -- see the port +# plan's Phase-0 measurement). +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini +# for the native workflow: RUN_DATETIME -> False (see +# config_tile_Git_symlink.ini's header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Ng + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +# Pinned explicitly: leaving this unset triggers ngmix.py's +# pixel_scale_from_wcs() fallback, which crashes on the merged-headers +# store's 'TILE_ID' metadata key ('str' object has no attribute 'values') -- +# a real upstream bug in a code path real-data configs never exercise +# (their PIXEL_SCALE is always pinned too, to 0.186). SKiLLS grid sims are +# rendered at 0.185"/px (Cail, fix/imsims-pixel-scale-0.185[-v2]). +PIXEL_SCALE = 0.185 + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner,run_sp_tile_fpsf:fake_psf_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner + +FILE_PATTERN = sexcat, image_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# No background subtraction for image sims (background not simulated) +BKG_SUB = False + +SAVE_BATCH = 1000 + +ID_OBJ_MIN = -1 +ID_OBJ_MAX = -1 diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini new file mode 100644 index 00000000..bc44b971 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -0,0 +1,167 @@ +# ShapePipe configuration file for tile, from detection up to shape +# measurement. Fake PSF model. +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini +# for the native workflow: RUN_DATETIME -> False, explicit (see +# config_tile_Git_symlink.ini's header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_ViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names + +MODULE = vignetmaker_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner + +FILE_PATTERN = sexcat, CFIS_simu_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner +ME_IMAGE_PATTERN = flag, image, weight + + +[VIGNETMAKER_RUNNER_RUN_3] + +# Cut per-object coadd-frame segmentation stamps from the tile SExtractor +# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, +# SEGMENTATION). Integer labels, no interpolation, zero-padded -- CLASSIC mode +# guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN +# centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg +# stamp directly for uberseg neighbour masking (shapepipe#776). + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner + +FILE_PATTERN = sexcat, segmentation + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# CLASSIC: cut stamps at object positions in the coadd (tile) frame. +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels -- MUST equal the coadd VIGNET size (51) so the seg +# stamp overlays the galaxy stamp on an identical grid. +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = seg diff --git a/workflow/image_sims/native/config/cfis_image_sims/config_tile_Sx.ini b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Sx.ini new file mode 100644 index 00000000..9d1771e2 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/config_tile_Sx.ini @@ -0,0 +1,123 @@ +# ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. +# +# Forked from shapepipe/example/cfis_image_sims/config_tile_Sx.ini for the +# native workflow: RUN_DATETIME -> False, explicit (see +# config_tile_Git_symlink.ini's header comment for why). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner + +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, SEGMENTATION + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/image_sims/native/config/cfis_image_sims/default.conv b/workflow/image_sims/native/config/cfis_image_sims/default.conv new file mode 100644 index 00000000..2590b9cb --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/workflow/image_sims/native/config/cfis_image_sims/default_noimaflags.param b/workflow/image_sims/native/config/cfis_image_sims/default_noimaflags.param new file mode 100644 index 00000000..b251f5b7 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/default_noimaflags.param @@ -0,0 +1,65 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/image_sims/native/config/cfis_image_sims/default_tile.sex b/workflow/image_sims/native/config/cfis_image_sims/default_tile.sex new file mode 100644 index 00000000..ff3b2521 --- /dev/null +++ b/workflow/image_sims/native/config/cfis_image_sims/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/image_sims/native/rules/exposure.smk b/workflow/image_sims/native/rules/exposure.smk new file mode 100644 index 00000000..21280c59 --- /dev/null +++ b/workflow/image_sims/native/rules/exposure.smk @@ -0,0 +1,69 @@ +"""Exposure chain, step 2 of the port: per exposure. + + exp_get_images -> exp_split + +NO exp_psf here, unlike shapepipe/workflow/rules/exposure.smk -- image sims +use a known, injected PSF (fake_psf_runner), not a fitted PSFEx model. +Gen-2's own job dispatch treats exposure-level PSF (job 64) as a pure +placeholder for image_sims ("fake PSF runs as part of job 512"); the real +fake_psf call happens at the TILE level, added in tile.smk (the next step), +reading the tile's own sexcat rather than any exposure product. + +Retrieval is symlink (config_exp_Gie_symlink.ini), not VOSpace. +""" + +rule exp_get_images: + output: + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" + log: + f"{EXP_DIR}/logs/exp_get_images.json" + params: + pre = lambda wc: unit_pre("exp_get_images", wc.exp) + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("exp_get_images", "config_exp_Gie_symlink.ini") + + +rule exp_split: + input: + rules.exp_get_images.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_split.json" + log: + f"{EXP_DIR}/logs/exp_split.json" + params: + pre = lambda wc: unit_pre("exp_split", wc.exp) + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("exp_split", "config_exp_Sp.ini") + + +# --- reclamation (see the Snakefile's module docstring) --------------------- +# Reuses shapepipe/workflow/scripts/clean_exposure.py UNMODIFIED. Input is +# every IN-SCOPE consuming tile's tile_vignets manifest -- vignets is the +# last stage that reads exposure products here too (tile_ngmix/tile_make_cat +# read only tile-level output) -- so writer, then readers, then cleaner, is +# the whole ordering argument, DAG-enforced. +rule clean_exposure: + input: + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp) if t in READY_SET] + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)) + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SHAPEPIPE_SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/image_sims/native/rules/prepare.smk b/workflow/image_sims/native/rules/prepare.smk new file mode 100644 index 00000000..08673e7b --- /dev/null +++ b/workflow/image_sims/native/rules/prepare.smk @@ -0,0 +1,94 @@ +"""PREPARE chain, step 1 of the port: one static chain per tile. + + tile_get_images -> tile_uncompress (faked) -> tile_find_exposures + +See the Snakefile's module docstring for what differs from +shapepipe/workflow/rules/prepare.smk and why. +""" + +rule tile_get_images: + output: + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" + log: + f"{TILE_DIR}/logs/tile_get_images.json" + params: + pre = lambda wc: unit_pre("tile_get_images", wc.tile) + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("tile_get_images", "config_tile_Git_symlink.ini") + + +def _tile_uncompress_shell(stage="tile_uncompress"): + """Not a shapepipe_run call: the image-sims weight is already + uncompressed, so this fakes the uncompress_fits_runner output by + symlinking it in directly (mirrors run_job_sp_canfar_v2.0.bash's job-2 + special case for image_sims), then goes through the same completeness + contract as every other stage with --job-rc 0. + + Also registers the fake run in $SP_RUN/output/log_run_sp.txt, in the + same " " format a real shapepipe_run call appends -- + ShapePipe's own input-resolution ("last:uncompress_fits_runner", read by + config_tile_Sx.ini) walks that registry, not the filesystem, so a run + that never calls shapepipe_run must still register itself there or every + downstream `last:`/exact-name reference to it fails with "No previous + run of module 'uncompress_fits_runner' found" even though the output + directory genuinely exists. + """ + _, subdir = STAGE_DIR[stage] + return ( + "{params.pre}\n" + "rc=0\n" + f'run_dir="$SP_RUN/output/{subdir}"\n' + 'out="$run_dir/uncompress_fits_runner/output"\n' + 'mkdir -p "$out"\n' + 'weight_src="$SP_DIR/input_tiles/{params.weight_name}"\n' + 'if [ -e "$weight_src" ]; then\n' + ' ln -sf "$weight_src" "$out/{params.weight_name}"\n' + ' printf "%s %s\\n" "$run_dir" uncompress_fits_runner >> "$SP_RUN/output/log_run_sp.txt"\n' + "else\n" + ' echo "tile_uncompress: missing $weight_src" >&2\n' + " rc=1\n" + "fi\n" + f"python {SCRIPTS}/completeness_sims.py check {stage} {{output.manifest}}" + ' --log {log} --job-rc "$rc" || rc=1\n' + "exit $rc\n" + ) + + +rule tile_uncompress: + input: + rules.tile_get_images.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" + log: + f"{TILE_DIR}/logs/tile_uncompress.json" + params: + pre = lambda wc: unit_pre("tile_uncompress", wc.tile), + weight_name = lambda wc: f"CFIS_simu_weight-{tile_dash(wc.tile)}.fits" + threads: 1 + resources: + mem_mb = 2000, + runtime = 15 + shell: + _tile_uncompress_shell() + + +rule tile_find_exposures: + input: + rules.tile_uncompress.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" + log: + f"{TILE_DIR}/logs/tile_find_exposures.json" + params: + pre = lambda wc: unit_pre("tile_find_exposures", wc.tile) + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 + shell: + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/image_sims/native/rules/tile.smk b/workflow/image_sims/native/rules/tile.smk new file mode 100644 index 00000000..f0ffa10c --- /dev/null +++ b/workflow/image_sims/native/rules/tile.smk @@ -0,0 +1,190 @@ +"""Tile post-chain, step 3 of the port: gather exposures, then detect / PSF / +shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> {tile_fake_psf, tile_vignets} -> tile_ngmix -> tile_make_cat + +Differences from shapepipe/workflow/rules/tile.smk, and why -- see also the +Snakefile's module docstring: + + * tile_fake_psf has no real-data counterpart: image sims use a known, + injected PSF (fake_psf_runner, reading the tile's own sexcat from + tile_detect), not a fitted PSFEx model. It is a SIBLING of tile_vignets + (both depend only on tile_detect), not a prerequisite of it -- Gen-2 ran + them sequentially only because its bash dispatch is inherently serial; + ngmix is the first stage that actually needs both. + * No tile_merge_cats or ngmix chunking: Gen-2 ran ngmix as ONE call per + tile (config_tile_Ng_batch_psfex_sx.ini), so tile_make_cat reads + tile_ngmix's output directly. No node-local staging or group fusion + either -- everything here runs on shared storage, plain ungrouped + rules. Both are Phase-3 concerns once this chain is verified correct. + * tile_detect depends on tile_merge_headers (config_tile_Sx.ini reads + log_exp_headers as one of its inputs), which real data's tile_detect + does not -- an image-sims-specific ordering. +""" + +rule tile_exp_forest: + input: + tile_exp_split + output: + forest = directory(f"{TILE_DIR}/exp_forest") + params: + cmd = lambda wc: (f"python {SCRIPTS}/build_forest_sims.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_JSON}") + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 + shell: + "{params.cmd} --forest {output.forest}" + + +rule tile_merge_headers: + input: + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" + log: + f"{TILE_DIR}/logs/tile_merge_headers.json" + params: + pre = lambda wc: unit_pre("tile_merge_headers", wc.tile, + forest=forest_dir(wc.tile)) + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") + + +rule tile_detect: + input: + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_detect.json" + log: + f"{TILE_DIR}/logs/tile_detect.json" + params: + pre = lambda wc: unit_pre("tile_detect", wc.tile) + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 + shell: + sp_shell("tile_detect", "config_tile_Sx.ini") + + +rule tile_fake_psf: + input: + sx = rules.tile_detect.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_fake_psf.json" + log: + f"{TILE_DIR}/logs/tile_fake_psf.json" + params: + pre = lambda wc: unit_pre("tile_fake_psf", wc.tile, + pre_run=[f"export PSF_DICT='{PSF_DICT}'"]) + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 + shell: + sp_shell("tile_fake_psf", "config_exp_psfex.ini") + + +rule tile_vignets: + input: + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + output: + manifest = f"{TILE_DIR}/manifests/tile_vignets.json" + log: + f"{TILE_DIR}/logs/tile_vignets.json" + params: + pre = lambda wc: unit_pre("tile_vignets", wc.tile, + forest=forest_dir(wc.tile)) + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 60 + shell: + sp_shell("tile_vignets", "config_tile_PiViVi_canfar_sx.ini") + + +rule tile_ngmix: + input: + sx = rules.tile_detect.output.manifest, + fpsf = rules.tile_fake_psf.output.manifest, + vignets = rules.tile_vignets.output.manifest, + mh = rules.tile_merge_headers.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_ngmix.json" + log: + f"{TILE_DIR}/logs/tile_ngmix.json" + params: + pre = lambda wc: unit_pre("tile_ngmix", wc.tile) + threads: 4 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = lambda wc, attempt: 180 * attempt + shell: + sp_shell("tile_ngmix", "config_tile_Ng_batch_psfex_sx.ini") + + +rule tile_make_cat: + input: + sx = rules.tile_detect.output.manifest, + fpsf = rules.tile_fake_psf.output.manifest, + ng = rules.tile_ngmix.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{PROD_TILE_DIR}/final_cat-{{tile}}.fits", + log: + f"{TILE_DIR}/logs/tile_make_cat.json" + params: + pre = lambda wc: unit_pre("tile_make_cat", wc.tile) + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 30 + shell: + # Publish guarded on rc, same as the real design: a job whose + # shapepipe_run died must never publish a catalogue for a manifest + # completeness.py is about to refuse to write. + sp_shell("tile_make_cat", "config_tile_Mc_psfex.ini", + post='if [ $rc -eq 0 ]; then\n' + ' mkdir -p "$(dirname {output.final_cat})"\n' + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_tile_Mc_psfex' + '/make_cat_runner/output/final_cat*.fits | head -1)" ' + '{output.final_cat}\n' + 'fi\n') + + +# --- reclamation (see the Snakefile's module docstring) --------------------- +# Reuses shapepipe/workflow/scripts/clean_tile.py UNMODIFIED. Input is this +# tile's final_cat on the persistent root -- a tile's store has no consumer +# outside itself, so unlike clean_exposure there is no eligibility test, only +# the ordering edge. clean_tile.py's own survivor list already matches our +# layout byte for byte (tile_vignets.json, tile_find_exposures.json, and +# output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-.txt -- +# verified: our own Fe output uses exactly that dashed name already). +rule clean_tile: + input: + lambda wc: final_cat(wc.tile) + output: + tombstone = f"{TILE_DIR}/cleaned.json" + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SHAPEPIPE_SCRIPTS}/clean_tile.py" + " --tile-dir $(dirname {output.tombstone}) --tile {wildcards.tile}" + " --tombstone {output.tombstone}" diff --git a/workflow/image_sims/native/run.example.yaml b/workflow/image_sims/native/run.example.yaml new file mode 100644 index 00000000..8e83b325 --- /dev/null +++ b/workflow/image_sims/native/run.example.yaml @@ -0,0 +1,44 @@ +# Template for a run config -- copy this OUTSIDE the repo (e.g. into your own +# run dir: ~/v2.0/image_sims/new_workflow/run.yaml) and edit every value +# below, then point run.sh at it: +# +# ./run.sh prepare -c ~/v2.0/image_sims/new_workflow/run.yaml +# ./run.sh compute -c ~/v2.0/image_sims/new_workflow/run.yaml +# +# Nothing here has a default in config.yaml -- these are exactly the values +# that differ from run to run / user to user, so they don't belong tracked +# in git under sp_validation. + +# One tile per line, "IDra.IDdec". Also keep this file itself outside the +# repo (e.g. beside your run config) -- see tiles.example.txt. +tile_list: /path/to/your/tiles.txt + +# The sim branch this campaign covers. Each branch is its own independent +# SKiLLS image tree -- no cross-branch exposure sharing, so each branch +# wants its own run config / run_dir like this one. +sim: 1z2z_grid_1 + +inputs: + tiles: /n09data/hervas/skills_out/1z2z_grid_1/images/SP_tiles + exposures: /n09data/hervas/skills_out/1z2z_grid_1/images/SP_exp + +outputs: + run_dir: /path/to/your/run/dir + +# Currently the shared pipeline container (see my_config_grid_1.yaml's +# sif_pipeline) -- override if you're testing your own image build. +container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif + +# Pickled PSF dictionary read by tile_fake_psf (the known, injected PSF +# image sims use instead of a fitted PSFEx model). +psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle + +# Used to call shapepipe/workflow/scripts/{clean_exposure,clean_tile}.py +# unmodified (see config.yaml for the clean:/clean_tiles: switches). +shapepipe_repo: /n17data/mkilbing/astro/repositories/github/shapepipe + +# Reclamation is off in config.yaml's defaults; opt in here if disk matters +# for this run (it does once you're past one tile -- exposures ARE shared +# across neighbouring tiles). +# clean: true +# clean_tiles: true diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh new file mode 100755 index 00000000..49f710b5 --- /dev/null +++ b/workflow/image_sims/native/run.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Launcher for the native image-sims workflow -- the sp_validation-side +# analogue of shapepipe/workflow/bin/sp. Runs the two SP_PHASE invocations +# (prepare, compute) that a static Snakemake DAG needs because the exposure +# list is only known after tile_find_exposures actually runs -- see the +# Snakefile's module docstring and CLAUDE.md's SP_PHASE explanation. +# +# A run config is REQUIRED (-c/--config): everything that changes from run +# to run or user to user (tile_list, sim, inputs, the run dir, the +# container) has no default in the committed config.yaml -- see +# run.example.yaml for the template. Keep your actual run config and tile +# list OUTSIDE this repo, in your own run dir. +# +# Usage: +# ./run.sh -c my_run.yaml # both phases, in order +# ./run.sh prepare -c my_run.yaml [-n] # just PREPARE +# ./run.sh compute -c my_run.yaml [-n] # just COMPUTE +set -euo pipefail + +usage() { + echo "Usage: $0 [prepare|compute] -c [extra snakemake args]" >&2 + exit 1 +} + +# --- argument parsing (before the cd below, so a relative -c path resolves +# against the CALLER's directory, not this script's) ------------------------- +PHASE_ARG="both" +if [[ "${1:-}" == "prepare" || "${1:-}" == "compute" ]]; then + PHASE_ARG=$1 + shift +fi + +RUN_CONFIG="" +EXTRA_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + -c|--config) RUN_CONFIG=${2:-}; shift 2 ;; + -h|--help) usage ;; + *) EXTRA_ARGS+=("$1"); shift ;; + esac +done + +[[ -z "$RUN_CONFIG" ]] && { echo "Missing -c/--config " >&2; usage; } +[[ -f "$RUN_CONFIG" ]] || { echo "run config not found: $RUN_CONFIG" >&2; exit 1; } +RUN_CONFIG=$(readlink -f "$RUN_CONFIG") + +cd "$(dirname "${BASH_SOURCE[0]}")" + +# --- cluster / container settings, shared by both phases ------------------- +PARTITION=comp +ACCOUNT=cusers +EXCLUDE=n17,n09,n36 # flaky candide nodes (n17 mount issues, n09 no + # internet, n36) -- matches sp_validation's + # own workflow/profiles/candide/config.yaml +BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +SHAPEPIPE_SRC=/n17data/mkilbing/astro/repositories/github/shapepipe/src + +APPTAINER_ARGS="--cleanenv --bind $BIND --env PYTHONPATH=$SHAPEPIPE_SRC" + +# --- per-phase resources ----------------------------------------------------- +# PREPARE is three light rules per tile (Git/Uz/Fe). COMPUTE now runs the +# WHOLE exposure + tile chain in one srun allocation (local `--cores` +# execution, not the SLURM executor -- rule-level mem_mb/runtime in the +# .smk files are bookkeeping only until Phase 3), so its wall clock must +# cover the slowest chain, not just one rule: the existing Gen-2 run of the +# reference tile (240.291) took ~80 min end to end, ngmix alone ~54 min. +# 2h leaves real margin; revisit once more tiles/branches are added. +declare -A CPUS=( [prepare]=2 [compute]=8 ) +declare -A MEM=( [prepare]=8G [compute]=16G ) +declare -A TIME=( [prepare]=00:15:00 [compute]=02:00:00 ) +declare -A TARGET=( [prepare]="prepare_all_tiles" [compute]="" ) + +# sbatch, not srun: srun's behavior depends on whether the CALLING shell is +# already inside a SLURM allocation -- it then runs as a STEP of that job, +# constrained by whatever resources the outer job happens to have (bitten us +# twice: "job has expired" from a stale outer allocation, then "Job step's +# --cpus-per-task value exceeds that of job (8 > 2)" from a live but smaller +# one). sbatch always submits a brand-new, independent job regardless of the +# submitting shell's own SLURM_JOB_ID, so it can't nest. --wait blocks until +# the job finishes (same UX as srun); since sbatch writes output to a file +# rather than the terminal, this tails that file live and cleans up after. +run_phase() { + local phase=$1 + echo "=== SP_PHASE=$phase (config: $RUN_CONFIG) ===" + + local batch_script logfile + batch_script=$(mktemp "/tmp/im_sims_${phase}_XXXXXX.sh") + logfile="${batch_script%.sh}.log" + cat > "$batch_script" </dev/null; do sleep 0.5; done + tail -f "$logfile" & + local tail_pid=$! + + wait "$sbatch_pid"; local rc=$? + kill "$tail_pid" 2>/dev/null || true + rm -f "$batch_script" "$logfile" + return $rc +} + +case "$PHASE_ARG" in + prepare) run_phase prepare ;; + compute) run_phase compute ;; + both) run_phase prepare && run_phase compute ;; +esac diff --git a/workflow/image_sims/native/scripts/build_forest_sims.py b/workflow/image_sims/native/scripts/build_forest_sims.py new file mode 100644 index 00000000..ba90abf8 --- /dev/null +++ b/workflow/image_sims/native/scripts/build_forest_sims.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the $SP_EXP view). + +Forked from shapepipe/workflow/scripts/build_forest.py: identical mechanism +and rationale (see that file's docstring), except the tile->exposure lookup +reads the native workflow's plain-JSON index instead of a sqlite +run_index.sqlite (see the Snakefile's module docstring for why JSON is +enough here). +""" + +import argparse +import json +import shutil +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + index = json.loads(args.index.read_text()) + exps = index.get(args.tile, []) + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e[:2] / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape + dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree -- unlink() + # raises IsADirectoryError on it and would kill the job. + if dst.is_symlink() or dst.exists(): + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest_sims] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/image_sims/native/scripts/completeness_sims.py b/workflow/image_sims/native/scripts/completeness_sims.py new file mode 100644 index 00000000..be5b752f --- /dev/null +++ b/workflow/image_sims/native/scripts/completeness_sims.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""The count-based completeness table for the NATIVE image-sims workflow. + +Forked from shapepipe/workflow/scripts/completeness.py (same CLI, same +manifest/log contract -- see that file's docstring for the full design +rationale, which is unchanged here). Only the two module-level tables below +differ: COMPLETENESS and STAGE_DIR are image-sims-specific, since the real +one is keyed to real-data stage names, counts and RUN_NAMEs. + +Being built up stage by stage as the port proceeds -- now covers the full +chain through tile_make_cat. No exp_psf: image sims use a known injected PSF +(tile-level fake_psf, in tile.smk), not a fitted PSFEx model. No +tile_merge_cats: unlike the real design's N-chunk ngmix, image sims run +ngmix as one call (see config_tile_Ng_batch_psfex_sx.ini), so tile_make_cat +reads it directly. + +The tile.smk counts below (tile_detect, tile_fake_psf, tile_vignets, +tile_ngmix, tile_make_cat) are PROVISIONAL -- carried over from the +real-data table where the chain is structurally similar, or a best guess +where it isn't (tile_fake_psf, tile_vignets' run_2/run_3). Calibrate them +against the first real run's actual output counts and correct here, the +same way the real table documents being "verified against" specific +campaigns. +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +# stage -> {runner_subdir: {expect, [warn], [subpath]}} +COMPLETENESS = { + # --- tile prepare --- + # get_images: symlink retrieve, tile image + weight = 2 files (matches the + # real-data nibi-flavor count for the same RETRIEVE=symlink configs). + "tile_get_images": {"get_images_runner": dict(expect=2)}, + # tile_uncompress is FAKED for image sims (the sim weight is already + # uncompressed -- rules/prepare.smk symlinks it into a synthetic + # uncompress_fits_runner/output/ dir rather than calling shapepipe_run), + # but the manifest/completeness contract is kept identical so downstream + # rules and tooling can't tell the difference. + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1)}, + + # --- exposure chain --- + # symlink retrieve, image+weight+flag = 3 files (Gen-2's own + # run_job_sp_canfar_v2.0.bash used n_exp=3 for image_sims vs 6 for data). + "exp_get_images": {"get_images_runner": dict(expect=3)}, + # Same threshold as real data's split_exp_runner (Gen-2's dispatch does + # not branch on type for job 16): 40 CCDs x 3 outputs + 1. + "exp_split": {"split_exp_runner": dict(expect=121)}, + + # --- tile post (PROVISIONAL -- see module docstring) --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2)}, + # fake_psf_runner: one aggregate output (a galaxy_psf sqlite) per tile. + "tile_fake_psf": {"fake_psf_runner": dict(expect=1)}, + "tile_vignets": { + "vignetmaker_runner_run_1": dict(expect=1), + # 3, not the real design's 5: image/weight/flag only -- image sims + # has no simulated background, so no background/background_rms + # multi-epoch vignet (config_tile_Sx.ini: "no background subtraction + # for image sims"). + "vignetmaker_runner_run_2": dict(expect=3), + # RUN_3 (segmentation vignet) has no real-data counterpart at all. + "vignetmaker_runner_run_3": dict(expect=1), + }, + "tile_ngmix": {"ngmix_runner": dict(expect=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1)}, +} + + +def count_products(run_dir, runner, spec): + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + n = 0 + try: + with os.scandir(out) as entries: + for e in entries: + if not e.is_symlink() or os.path.exists(e.path): + n += 1 + except OSError: + return 0 + return n + + +def check_counts(stage, run_dir): + table = COMPLETENESS[stage] + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + warn = spec.get("warn", False) + details.append((runner, n, spec["expect"], warn)) + if not warn and n < spec["expect"]: + ok = False + return ok, details + + +# stage -> (level, run_sp_ dir under $SP_RUN/output/). Every ini this +# table names is a fork under config/cfis_image_sims/ with RUN_DATETIME=False, +# so the path is fixed -- no run-log resolution needed. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_fake_psf": ("tile", "run_sp_tile_fpsf"), + "tile_vignets": ("tile", "run_sp_tile_ViVi"), + "tile_ngmix": ("tile", "run_sp_tile_Ng"), + "tile_make_cat": ("tile", "run_sp_tile_Mc_psfex"), +} + +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 +MAX_TAIL_LINES = 120 +MAX_REASONS = 3 + + +def _normalise(line: str) -> str: + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, "level": level, "unit": unit, + "run_dir": str(run_dir), "stage_dir": str(stage_dir), + "runners": {}, "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "expect": 1, "warn": False, + "status": "failed", "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_counts(stage, stage_dir) + short = False + for runner, n, expect, warn in details: + below = n < expect + if below: + short = True + status = "complete" if not below else "warn" if warn else "failed" + manifest["runners"][runner] = { + "found": n, "expect": expect, "warn": warn, "status": status, + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "expect": expect, + "warn": warn, "status": status, + "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def write_if_changed(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != text: + path.write_text(text) + + +def _unit_from_run_dir(run_dir): + return Path(str(run_dir)).name or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="Image-sims per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--log", type=Path, required=True) + c.add_argument("--run-dir", type=Path, default=None) + c.add_argument("--unit", default=None) + c.add_argument("--stage-dir", default=None) + c.add_argument("--job-rc", type=int, default=0) + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness_sims] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_run_dir(run_dir) + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + + if args.job_rc != 0: + ok = False + manifest["status"] = "failed" + manifest["job_rc"] = args.job_rc + manifest["failures"].append({ + "runner": "shapepipe_run", "found": 0, "expect": 1, + "warn": False, "status": "failed", + "reasons": [f"shapepipe_run exited {args.job_rc} (counts met expect)"], + }) + + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + write_if_changed(args.log, text) + if ok: + write_if_changed(args.manifest, text) + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "failed": "<-- BELOW expect"} + print(f"[completeness_sims] {runner}: {r['found']}/{r['expect']} " + f"{tag[r['status']]}", file=sys.stderr) + print(f"[completeness_sims] {args.stage} {unit}: {manifest['status']} " + f"-> {args.log}" + (f" + {args.manifest}" if ok else ""), file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness_sims] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/image_sims/native/scripts/status_sims.py b/workflow/image_sims/native/scripts/status_sims.py new file mode 100644 index 00000000..c0200778 --- /dev/null +++ b/workflow/image_sims/native/scripts/status_sims.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Disk-scan status report for the native image-sims workflow. + +Not a DAG node (same reasoning as shapepipe/workflow/scripts/run_report.py: +a report that's part of the DAG would itself be poisoned by the failures it +must enumerate). Run any time, mid-run or after: + + python scripts/status_sims.py + +For each tile/exposure and each stage, a manifest means complete, a log +with no manifest means failed, and neither means not yet attempted -- the +same three-way read completeness.py's own contract establishes. +""" + +import argparse +import json +import sys +from pathlib import Path + +TILE_STAGES = [ + "tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_fake_psf", "tile_vignets", + "tile_ngmix", "tile_make_cat", +] +EXP_STAGES = ["exp_get_images", "exp_split"] + +# The Gen-2 RUN_NAME suffixes (run_sp_tile_Git, run_sp_exp_Gie, ...) -- +# shorter and more familiar than the stage names for a wide table. +STAGE_ABBR = { + "tile_get_images": "Git", + "tile_uncompress": "Uz", + "tile_find_exposures": "Fe", + "tile_merge_headers": "Mh_exp", + "tile_detect": "Sx", + "tile_fake_psf": "fpsf", + "tile_vignets": "ViVi", + "tile_ngmix": "Ng", + "tile_make_cat": "Mc", + "exp_get_images": "Gie", + "exp_split": "Sp", +} + +SYMBOL = {"complete": ".", "failed": "X", "missing": " "} + + +def stage_status(unit_dir: Path, stage: str) -> str: + if (unit_dir / "manifests" / f"{stage}.json").exists(): + return "complete" + if (unit_dir / "logs" / f"{stage}.json").exists(): + return "failed" + return "missing" + + +def report_finished(run_dir: Path, kind: str, stages: list) -> None: + """--very-short: the abbreviated header, and ONE row below it -- how many + units have COMPLETED each step -- instead of one row per unit.""" + root = run_dir / kind + units = sorted(p for p in root.glob("*/*") if p.is_dir()) if root.is_dir() else [] + print(f"{len(units)} {kind}") + if not units: + return + + header = f"{'':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) + print(header) + print("-" * len(header)) + + counts = {s: 0 for s in stages} + for unit_dir in units: + for s in stages: + if stage_status(unit_dir, s) == "complete": + counts[s] += 1 + row = f"{'complete':<12}" + "".join(f"{counts[s]:>8}" for s in stages) + print(row) + + +def report_units(run_dir: Path, kind: str, stages: list, *, short: bool = False) -> None: + root = run_dir / kind + units = sorted(p for p in root.glob("*/*") if p.is_dir()) if root.is_dir() else [] + if not units: + print(f"No {kind} units found under {root}") + return + + counts = {s: {"complete": 0, "failed": 0, "missing": 0} for s in stages} + + header = f"{'unit':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) + print(header) + print("-" * len(header)) + + for unit_dir in units: + unit = unit_dir.name + row = f"{unit:<12}" + for s in stages: + st = stage_status(unit_dir, s) + counts[s][st] += 1 + row += f"{SYMBOL[st]:>8}" + print(row) + + # --short: just the per-unit table above, skip the aggregate counts below. + if short: + print() + return + + print() + print(f"{len(units)} {kind} total") + for s in stages: + c = counts[s] + print(f" {STAGE_ABBR[s]:<8} complete={c['complete']:<4} failed={c['failed']:<4} " + f"not-yet-run={c['missing']}") + print() + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("run_dir", type=Path) + p.add_argument("-s", "--short", action="store_true", + help="per-tile/per-exposure table only, no aggregate " + "'N units total' / per-stage count summary") + p.add_argument("-S", "--very-short", action="store_true", + help="no per-unit table at all -- just 'N/M finished' " + "for tiles and exposures, once each (overrides -s)") + args = p.parse_args(argv) + + print(f"=== {args.run_dir} ===") + + if args.very_short: + report_finished(args.run_dir, "tiles", TILE_STAGES) + report_finished(args.run_dir, "exp", EXP_STAGES) + return 0 + + print() + print("--- tiles ---") + report_units(args.run_dir, "tiles", TILE_STAGES, short=args.short) + print("--- exposures ---") + report_units(args.run_dir, "exp", EXP_STAGES, short=args.short) + + final_cats = list(args.run_dir.glob("tiles/*/*/final_cat-*.fits")) + print(f"final_cat produced: {len(final_cats)}") + if not args.short: + for f in final_cats: + print(f" {f}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/image_sims/native/tiles.example.txt b/workflow/image_sims/native/tiles.example.txt new file mode 100644 index 00000000..1922554c --- /dev/null +++ b/workflow/image_sims/native/tiles.example.txt @@ -0,0 +1 @@ +240.291 From 9ba352fb7984138c0b52394ec8b700ebc8fb9203 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sun, 13 Sep 2026 20:59:14 +0200 Subject: [PATCH 13/36] image_sims/native: stop writing batch scripts to /tmp run.sh's mktemp targeted /tmp directly -- a small node-local disk shared by every job/user on the node (candide policy), and this runs on the login node besides. Use a gitignored .batch/ dir alongside the workflow instead. --- workflow/image_sims/native/.gitignore | 1 + workflow/image_sims/native/run.sh | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 workflow/image_sims/native/.gitignore diff --git a/workflow/image_sims/native/.gitignore b/workflow/image_sims/native/.gitignore new file mode 100644 index 00000000..c5c8bf7f --- /dev/null +++ b/workflow/image_sims/native/.gitignore @@ -0,0 +1 @@ +.batch/ diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh index 49f710b5..67e88fbd 100755 --- a/workflow/image_sims/native/run.sh +++ b/workflow/image_sims/native/run.sh @@ -46,6 +46,15 @@ RUN_CONFIG=$(readlink -f "$RUN_CONFIG") cd "$(dirname "${BASH_SOURCE[0]}")" +# NEVER /tmp: it's a small node-local disk shared by every job/user on that +# node (candide policy -- see ~/dev/felt/AGENTS.md's "Scratch and temporary +# files on CANDIDE"); a stray file there can starve someone else's job, and +# this runs on the LOGIN node besides. These batch scripts/logs are tiny and +# short-lived (removed at the end of run_phase) -- .batch/ here, on network +# storage, is gitignored. +BATCH_DIR="$(pwd)/.batch" +mkdir -p "$BATCH_DIR" + # --- cluster / container settings, shared by both phases ------------------- PARTITION=comp ACCOUNT=cusers @@ -84,7 +93,7 @@ run_phase() { echo "=== SP_PHASE=$phase (config: $RUN_CONFIG) ===" local batch_script logfile - batch_script=$(mktemp "/tmp/im_sims_${phase}_XXXXXX.sh") + batch_script=$(mktemp -p "$BATCH_DIR" "im_sims_${phase}_XXXXXX.sh") logfile="${batch_script%.sh}.log" cat > "$batch_script" < Date: Sun, 13 Sep 2026 21:10:42 +0200 Subject: [PATCH 14/36] image_sims/native: batch scripts/logs go to network storage, not /scratch Node-local /scratch (the previous fix's instinct, per CLAUDE.md's per-job scratch pattern) doesn't actually work here: verified empirically that a login-node /scratch path is invisible from the compute node the job lands on, so both the job's own stdout file and a file it tried to write there silently never appeared. This script runs before the job exists (login node) and its log is written from a not-yet-chosen compute node, so it needs network storage -- CLAUDE.md's own fallback for anything that must be visible beyond one node -- not /scratch, not /tmp, and not the repository directory either. Moved to /n17data/$USER/.im_sims_batch/. --- workflow/image_sims/native/.gitignore | 1 - workflow/image_sims/native/run.sh | 22 +++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) delete mode 100644 workflow/image_sims/native/.gitignore diff --git a/workflow/image_sims/native/.gitignore b/workflow/image_sims/native/.gitignore deleted file mode 100644 index c5c8bf7f..00000000 --- a/workflow/image_sims/native/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.batch/ diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh index 67e88fbd..12052052 100755 --- a/workflow/image_sims/native/run.sh +++ b/workflow/image_sims/native/run.sh @@ -46,13 +46,21 @@ RUN_CONFIG=$(readlink -f "$RUN_CONFIG") cd "$(dirname "${BASH_SOURCE[0]}")" -# NEVER /tmp: it's a small node-local disk shared by every job/user on that -# node (candide policy -- see ~/dev/felt/AGENTS.md's "Scratch and temporary -# files on CANDIDE"); a stray file there can starve someone else's job, and -# this runs on the LOGIN node besides. These batch scripts/logs are tiny and -# short-lived (removed at the end of run_phase) -- .batch/ here, on network -# storage, is gitignored. -BATCH_DIR="$(pwd)/.batch" +# NOT /tmp (small node-local disk shared by every job/user on the node -- +# see ~/CLAUDE.md's "Scratch and temporary files on CANDIDE"), and NOT +# node-local /scratch either, despite that section's per-job pattern: this +# script is created on the LOGIN node before the job exists, and its log is +# written FROM whichever compute node SLURM schedules the job on -- and +# login-node /scratch is NOT visible there (verified empirically: a probe +# job's own stdout file and a file it tried to write both silently failed +# to appear). /scratch is the right tool for a job's own intermediates, +# computed and consumed on the one node that owns them; it's the wrong tool +# for something that must be visible before the job exists and from a node +# not yet chosen. That leaves network storage -- CLAUDE.md's own fallback +# ("anything that must survive the job goes to /n*data*/$USER/...") -- and +# NOT the repository directory either (same doc: "never write test outputs +# into the repository directory"), hence a dotdir outside the checkout. +BATCH_DIR="/n17data/$USER/.im_sims_batch" mkdir -p "$BATCH_DIR" # --- cluster / container settings, shared by both phases ------------------- From 30904defef7a38ba17cbdaf307318913c84057df Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 15:16:02 +0200 Subject: [PATCH 15/36] image_sims/native: 47h compute budget, not 2h TIME[compute]=02:00:00 was sized for the original single reference-tile run (~80 min end to end). COMPUTE runs the whole tile list in one --cores 8 allocation, so its wall clock has to cover the campaign, not one tile: at 40 tiles (1z2z_grid_1) the 2h cap timed out at 40/376 steps with no ngmix job finished (candide job 867554). 47h leaves margin under comp/pscomp's 48h MaxTime. --- workflow/image_sims/native/run.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh index 12052052..917ff332 100755 --- a/workflow/image_sims/native/run.sh +++ b/workflow/image_sims/native/run.sh @@ -76,15 +76,23 @@ APPTAINER_ARGS="--cleanenv --bind $BIND --env PYTHONPATH=$SHAPEPIPE_SRC" # --- per-phase resources ----------------------------------------------------- # PREPARE is three light rules per tile (Git/Uz/Fe). COMPUTE now runs the -# WHOLE exposure + tile chain in one srun allocation (local `--cores` -# execution, not the SLURM executor -- rule-level mem_mb/runtime in the -# .smk files are bookkeeping only until Phase 3), so its wall clock must -# cover the slowest chain, not just one rule: the existing Gen-2 run of the -# reference tile (240.291) took ~80 min end to end, ngmix alone ~54 min. -# 2h leaves real margin; revisit once more tiles/branches are added. +# WHOLE exposure + tile chain for the WHOLE tile list in one srun allocation +# (local `--cores` execution, not the SLURM executor -- rule-level +# mem_mb/runtime in the .smk files are bookkeeping only until Phase 3), so +# its wall clock must cover the campaign, not just one rule or one tile: the +# existing Gen-2 run of the reference tile (240.291) took ~80 min end to end, +# ngmix alone ~54 min, and 2h was sized for that ONE-tile run. At 40 tiles +# (1z2z_grid_1, 2026-09) a 2h cap timed out at 40/376 steps with not one +# ngmix job finished (867554) -- there just isn't enough wall clock in the +# one allocation for a campaign this size, regardless of the 8-core packing +# Snakemake's local scheduler does across tiles. 47h (comp/pscomp's MaxTime +# is 2-00:00:00, i.e. 48h) leaves margin under the cap while still fitting in +# one submission; revisit (and think about the SLURM executor instead of +# local `--cores`, matching shapepipe's own workflow) once the campaign grows +# past what 47h covers. declare -A CPUS=( [prepare]=2 [compute]=8 ) declare -A MEM=( [prepare]=8G [compute]=16G ) -declare -A TIME=( [prepare]=00:15:00 [compute]=02:00:00 ) +declare -A TIME=( [prepare]=00:15:00 [compute]=47:00:00 ) declare -A TARGET=( [prepare]="prepare_all_tiles" [compute]="" ) # sbatch, not srun: srun's behavior depends on whether the CALLING shell is From 07441ae8f09a32475433c97a09d55baa61873053 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 18:52:26 +0200 Subject: [PATCH 16/36] image_sims/native: exclude n23; shorten the time-limit comment The 47h compute job hung on n23 with zero CPU for ~30 min after tile_merge_headers; shapepipe's candide profile already excludes n23 for past job failures. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/image_sims/native/run.sh | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh index 917ff332..60654a64 100755 --- a/workflow/image_sims/native/run.sh +++ b/workflow/image_sims/native/run.sh @@ -66,30 +66,17 @@ mkdir -p "$BATCH_DIR" # --- cluster / container settings, shared by both phases ------------------- PARTITION=comp ACCOUNT=cusers -EXCLUDE=n17,n09,n36 # flaky candide nodes (n17 mount issues, n09 no - # internet, n36) -- matches sp_validation's - # own workflow/profiles/candide/config.yaml +# Flaky candide nodes: n17 mounts, n09 no internet, n23 hung jobs, n36. +EXCLUDE=n09,n17,n23,n36 BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data SHAPEPIPE_SRC=/n17data/mkilbing/astro/repositories/github/shapepipe/src APPTAINER_ARGS="--cleanenv --bind $BIND --env PYTHONPATH=$SHAPEPIPE_SRC" # --- per-phase resources ----------------------------------------------------- -# PREPARE is three light rules per tile (Git/Uz/Fe). COMPUTE now runs the -# WHOLE exposure + tile chain for the WHOLE tile list in one srun allocation -# (local `--cores` execution, not the SLURM executor -- rule-level -# mem_mb/runtime in the .smk files are bookkeeping only until Phase 3), so -# its wall clock must cover the campaign, not just one rule or one tile: the -# existing Gen-2 run of the reference tile (240.291) took ~80 min end to end, -# ngmix alone ~54 min, and 2h was sized for that ONE-tile run. At 40 tiles -# (1z2z_grid_1, 2026-09) a 2h cap timed out at 40/376 steps with not one -# ngmix job finished (867554) -- there just isn't enough wall clock in the -# one allocation for a campaign this size, regardless of the 8-core packing -# Snakemake's local scheduler does across tiles. 47h (comp/pscomp's MaxTime -# is 2-00:00:00, i.e. 48h) leaves margin under the cap while still fitting in -# one submission; revisit (and think about the SLURM executor instead of -# local `--cores`, matching shapepipe's own workflow) once the campaign grows -# past what 47h covers. +# COMPUTE runs the whole tile list in one --cores allocation, so its wall +# clock covers the campaign, not one tile (~80 min/tile). 47h: just under +# comp/pscomp's 48h MaxTime. declare -A CPUS=( [prepare]=2 [compute]=8 ) declare -A MEM=( [prepare]=8G [compute]=16G ) declare -A TIME=( [prepare]=00:15:00 [compute]=47:00:00 ) From a2bb376530a38a7d64e3193ed767375896d213a1 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 18:54:07 +0200 Subject: [PATCH 17/36] image_sims/native: also exclude n24/n25 (TmpDisk=0, like n23) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/image_sims/native/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh index 60654a64..3fffbbf1 100755 --- a/workflow/image_sims/native/run.sh +++ b/workflow/image_sims/native/run.sh @@ -66,8 +66,9 @@ mkdir -p "$BATCH_DIR" # --- cluster / container settings, shared by both phases ------------------- PARTITION=comp ACCOUNT=cusers -# Flaky candide nodes: n17 mounts, n09 no internet, n23 hung jobs, n36. -EXCLUDE=n09,n17,n23,n36 +# Flaky candide nodes: n17 mounts, n09 no internet, n36; n23-n25 have +# TmpDisk=0 (a job hung on n23 with zero CPU). +EXCLUDE=n09,n17,n23,n24,n25,n36 BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data SHAPEPIPE_SRC=/n17data/mkilbing/astro/repositories/github/shapepipe/src From ecaf94722728591ec72230c722628e74ed93d309 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 17 Sep 2026 07:12:29 +0200 Subject: [PATCH 18/36] image_sims/native: $variable expansion, shared defaults, status -c, no /tmp - scripts/run_config_sims.py: $name in any run-config value expands to the top-level scalar key `name` (my_base_dir, sim, ...), repeatedly; an unexpanded $name is an error rather than a literal directory. Used by the Snakefile and by status_sims.py. - config.yaml: container, psf_dict and shapepipe_repo become shared defaults, so a run config need not repeat them (it may still override them). - status_sims.py: optional -c/--config reads outputs.run_dir and outputs.products_dir from the run config; final_cat is counted on the products root, not under run_dir. - run.sh: TMPDIR is /n23data1/mkilbing/tmp, passed to snakemake (--default-resources tmpdir=) and into the container (--env TMPDIR, plus a bind), since --cleanenv drops the host value. Nothing writes to the node's /tmp any more, per ~/CLAUDE.md. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/image_sims/native/Snakefile | 19 ++++- workflow/image_sims/native/config.yaml | 7 +- workflow/image_sims/native/run.sh | 7 +- .../native/scripts/run_config_sims.py | 70 +++++++++++++++++++ .../image_sims/native/scripts/status_sims.py | 44 +++++++++--- 5 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 workflow/image_sims/native/scripts/run_config_sims.py diff --git a/workflow/image_sims/native/Snakefile b/workflow/image_sims/native/Snakefile index 98820d1e..50ddebfe 100644 --- a/workflow/image_sims/native/Snakefile +++ b/workflow/image_sims/native/Snakefile @@ -81,6 +81,7 @@ against the Gen-2 reference tile. import json import os +import re from pathlib import Path from snakemake.exceptions import WorkflowError @@ -98,14 +99,26 @@ from snakemake.exceptions import WorkflowError # never needs to repeat the operational defaults. configfile: str(Path(workflow.snakefile).parent / "config.yaml") +# $name in any config value is replaced by the top-level scalar key `name` +# (e.g. `my_base_dir`, `sim`); scripts/run_config_sims.py is the one +# implementation, shared with status_sims.py. +import sys as _sys +_sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import run_config_sims # noqa: E402 + +try: + run_config_sims.expand_vars(config) +except run_config_sims.ConfigError as _exc: + raise WorkflowError(f"run config: {_exc}") + _REQUIRED = ("tile_list", "sim", "inputs", "outputs", "container", "psf_dict", "shapepipe_repo") _missing = [k for k in _REQUIRED if k not in config] if _missing: raise WorkflowError( - f"Missing required run-config key(s) {_missing} -- these have no " - "default in config.yaml and must be set in a run config passed via " - "an extra --configfile (see run.example.yaml, and run.sh -c/--config)." + f"Missing required run-config key(s) {_missing} -- set them in the " + "run config passed via run.sh -c/--config, or (for values shared by " + "every run here) in config.yaml." ) for _k in ("tiles", "exposures"): if _k not in config["inputs"]: diff --git a/workflow/image_sims/native/config.yaml b/workflow/image_sims/native/config.yaml index ea555ce3..c67a1a7f 100644 --- a/workflow/image_sims/native/config.yaml +++ b/workflow/image_sims/native/config.yaml @@ -3,13 +3,18 @@ # literal fallback), so this file is their one home. # # Everything that changes from run to run or user to user (tile_list, sim, -# inputs.tiles/inputs.exposures, outputs.run_dir, container) has NO default +# inputs.tiles/inputs.exposures, outputs.run_dir) has NO default # here: it must come from a run config layered on top via a second # --configfile (later ones override earlier ones). See run.example.yaml for # the template, and run.sh's -c/--config flag. psf_model: psfex +# Shared by every run here; a run config may still override any of them. +container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif +psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle +shapepipe_repo: /n17data/mkilbing/astro/repositories/github/shapepipe + # Reclamation (off by default -- a run config opts in explicitly, same # convention as the real design's clean:/clean_tiles: keys). clean: # reclaims an exposure's store once every consuming tile has its vignets; diff --git a/workflow/image_sims/native/run.sh b/workflow/image_sims/native/run.sh index 3fffbbf1..43a155d4 100755 --- a/workflow/image_sims/native/run.sh +++ b/workflow/image_sims/native/run.sh @@ -72,7 +72,11 @@ EXCLUDE=n09,n17,n23,n24,n25,n36 BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data SHAPEPIPE_SRC=/n17data/mkilbing/astro/repositories/github/shapepipe/src -APPTAINER_ARGS="--cleanenv --bind $BIND --env PYTHONPATH=$SHAPEPIPE_SRC" +# TMPDIR must not be /tmp: it is a small node-local disk shared by every +# job on the node (~/CLAUDE.md). --cleanenv drops the host TMPDIR, so pass +# it in explicitly; snakemake gets the same path via --default-resources. +TMP_ROOT=/n23data1/mkilbing/tmp +APPTAINER_ARGS="--cleanenv --bind $BIND,$TMP_ROOT --env PYTHONPATH=$SHAPEPIPE_SRC --env TMPDIR=$TMP_ROOT" # --- per-phase resources ----------------------------------------------------- # COMPUTE runs the whole tile list in one --cores allocation, so its wall @@ -115,6 +119,7 @@ export SP_PHASE=$phase # swapping the two fixes it. --configfile is therefore placed LAST, with # only EXTRA_ARGS (always dash-prefixed flags like -n) after it. exec snakemake --cores ${CPUS[$phase]} --sdm apptainer \\ + --default-resources tmpdir=$TMP_ROOT \\ --apptainer-args "$APPTAINER_ARGS" \\ --rerun-triggers mtime params code software-env \\ -s Snakefile --directory . \\ diff --git a/workflow/image_sims/native/scripts/run_config_sims.py b/workflow/image_sims/native/scripts/run_config_sims.py new file mode 100644 index 00000000..401cc25f --- /dev/null +++ b/workflow/image_sims/native/scripts/run_config_sims.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Run-config handling for the native image-sims workflow. + +`$name` in any value is replaced by the top-level scalar key `name` +(`my_base_dir`, `sim`, ...), repeatedly, so one key can build on another. +Nothing else expands anything, so a leftover `$name` would become a literal +directory -- `expand_vars` raises instead. + +Used by the Snakefile (on snakemake's own merged config) and by +status_sims.py (which loads the files itself, via `load`). +""" + +import re +from pathlib import Path + +MAX_DEPTH = 5 + + +class ConfigError(ValueError): + pass + + +def expand_vars(config: dict) -> dict: + """Expand $name in every value of `config`, in place.""" + scalars = {k: v for k, v in config.items() + if isinstance(v, (str, int, float)) and not isinstance(v, bool)} + + def sub(value, depth=0): + if isinstance(value, str): + out = re.sub(r"\$(\w+)", + lambda m: str(scalars.get(m.group(1), m.group(0))), value) + return out if out == value or depth >= MAX_DEPTH else sub(out, depth + 1) + if isinstance(value, dict): + return {k: sub(v, depth) for k, v in value.items()} + if isinstance(value, list): + return [sub(v, depth) for v in value] + return value + + for key in list(config): + config[key] = sub(config[key]) + left = [k for k, v in config.items() if isinstance(v, str) and "$" in v] + if left: + raise ConfigError( + f"unexpanded $variable(s) in {left} -- a $name must name a " + f"top-level scalar key of the run config.") + return config + + +def load(run_config, config_yaml=None) -> dict: + """config.yaml, then `run_config` on top, then $name expansion.""" + import yaml + + if config_yaml is None: + config_yaml = Path(__file__).resolve().parent.parent / "config.yaml" + config = {} + for path in (config_yaml, run_config): + if path and Path(path).exists(): + with open(path) as f: + config.update(yaml.safe_load(f) or {}) + return expand_vars(config) + + +def run_dirs(run_config, config_yaml=None): + """Return `(run_dir, products_dir)` as Paths, products_dir defaulting to + run_dir exactly as the Snakefile does.""" + outputs = load(run_config, config_yaml).get("outputs") or {} + if "run_dir" not in outputs: + raise ConfigError(f"{run_config}: outputs.run_dir is not set") + run_dir = Path(outputs["run_dir"]) + return run_dir, Path(outputs.get("products_dir") or run_dir) diff --git a/workflow/image_sims/native/scripts/status_sims.py b/workflow/image_sims/native/scripts/status_sims.py index c0200778..582f616e 100644 --- a/workflow/image_sims/native/scripts/status_sims.py +++ b/workflow/image_sims/native/scripts/status_sims.py @@ -6,6 +6,7 @@ must enumerate). Run any time, mid-run or after: python scripts/status_sims.py + python scripts/status_sims.py -c my_run.yaml # dirs from the run config For each tile/exposure and each stage, a manifest means complete, a log with no manifest means failed, and neither means not yet attempted -- the @@ -109,9 +110,19 @@ def report_units(run_dir: Path, kind: str, stages: list, *, short: bool = False) print() +def final_cats(products_dir: Path) -> list: + """The published catalogues, on the products root (which is run_dir when + the run config sets no products_dir).""" + return sorted(products_dir.glob("tiles/*/*/final_cat-*.fits")) + + def main(argv=None) -> int: p = argparse.ArgumentParser(description=__doc__) - p.add_argument("run_dir", type=Path) + p.add_argument("run_dir", type=Path, nargs="?", + help="run directory; omit when passing -c") + p.add_argument("-c", "--config", type=Path, + help="run config (as given to run.sh): read outputs.run_dir " + "and outputs.products_dir from it, $variables expanded") p.add_argument("-s", "--short", action="store_true", help="per-tile/per-exposure table only, no aggregate " "'N units total' / per-stage count summary") @@ -120,23 +131,38 @@ def main(argv=None) -> int: "for tiles and exposures, once each (overrides -s)") args = p.parse_args(argv) - print(f"=== {args.run_dir} ===") + if args.config: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import run_config_sims + try: + run_dir, products_dir = run_config_sims.run_dirs(args.config) + except run_config_sims.ConfigError as exc: + p.error(f"{exc}") + elif args.run_dir: + run_dir = products_dir = args.run_dir + else: + p.error("give a run directory, or -c with a run config") + + print(f"=== {run_dir} ===") + if products_dir != run_dir: + print(f"=== products: {products_dir} ===") if args.very_short: - report_finished(args.run_dir, "tiles", TILE_STAGES) - report_finished(args.run_dir, "exp", EXP_STAGES) + report_finished(run_dir, "tiles", TILE_STAGES) + report_finished(run_dir, "exp", EXP_STAGES) + print(f"final_cat produced: {len(final_cats(products_dir))}") return 0 print() print("--- tiles ---") - report_units(args.run_dir, "tiles", TILE_STAGES, short=args.short) + report_units(run_dir, "tiles", TILE_STAGES, short=args.short) print("--- exposures ---") - report_units(args.run_dir, "exp", EXP_STAGES, short=args.short) + report_units(run_dir, "exp", EXP_STAGES, short=args.short) - final_cats = list(args.run_dir.glob("tiles/*/*/final_cat-*.fits")) - print(f"final_cat produced: {len(final_cats)}") + cats = final_cats(products_dir) + print(f"final_cat produced: {len(cats)}") if not args.short: - for f in final_cats: + for f in cats: print(f" {f}") return 0 From 4725caf2dfc2aacffacbc18bbc02a367a9876bd2 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 2 Oct 2026 09:03:12 +0200 Subject: [PATCH 19/36] workflow/bin/spv: launcher for the image-sims workflow, like ShapePipe's sp - spv run -c RUN_CONFIG [-t TARGET] [snakemake args]: the main config (workflow/image_sims/config.yaml) plus the user's run config, default target im_mbias. The code is snapshotted into /spv-state/code and the run executes out of it (image_sims.sp_validation_repo is set to the snapshot), so editing the checkout cannot reach running jobs. - On candide TMPDIR/APPTAINER_TMPDIR must be set outside /tmp and /scratch (the slurm executor exports the head's TMPDIR to every job, and Slurm falls back to the node's /tmp when it cannot be created); the snapshot's profile passes the same directory into the container. - spv status -c sp_*.yaml | DIR: disk-scan status of ShapePipe campaigns, workflow/scripts/status_sims.py (from the native prototype's script, which already read the unified layout; now with its own $name/${name} reader). - The image-sims Snakefile loads its config next to itself, not relative to the working directory; im_mbias_config.py reads repo provenance from a snapshot's snapshot.json when there is no .git. Co-Authored-By: Claude Opus 5.5 --- workflow/bin/spv | 211 +++++++++++ workflow/image_sims/Snakefile | 25 +- workflow/scripts/im_mbias_config.py | 26 +- workflow/scripts/status_sims.py | 563 ++++++++++++++++++++++++++++ 4 files changed, 801 insertions(+), 24 deletions(-) create mode 100755 workflow/bin/spv create mode 100644 workflow/scripts/status_sims.py diff --git a/workflow/bin/spv b/workflow/bin/spv new file mode 100755 index 00000000..6e722665 --- /dev/null +++ b/workflow/bin/spv @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# spv -- launcher for sp_validation's image-sims workflow; the counterpart of +# ShapePipe's workflow/bin/sp. +# +# Usage: +# spv run -c FILE [-t TARGET]... [SNAKEMAKE ARGS...] +# run the image-sims workflow for the run config FILE. +# Default target: im_mbias (init -> extract -> calibrate +# -> m-bias). Snapshots the code into the state dir first +# and runs out of the copy, so editing the checkout +# mid-run cannot reach the jobs. -n is a dry run. +# spv status -c FILE [FLAGS...] +# spv status DIR [FLAGS...] +# disk-scan status of ShapePipe campaigns: one sp run +# config, or every sp_*.yaml in DIR (default flag -S). +# Read-only; see workflow/scripts/status_sims.py --help. +# spv -c FILE -- [SNAKEMAKE ARGS...] +# snakemake on this run's existing snapshot (--unlock, +# --summary, ...); no new snapshot. +# +# Configs: workflow/image_sims/config.yaml is the main config (defaults); FILE +# is the user's run config, which must set image_sims.grids_base and +# image_sims.shapepipe_repo and may override anything else. Snakemake +# deep-merges the two. image_sims.sp_validation_repo is set by spv (to the +# snapshot) and need not be in FILE. +# +# Environment: +# SPV_PROFILE workflow/profiles/ (default candide) +# SPV_STATE_DIR snakemake state + code snapshot +# (default /spv-state) +# TMPDIR, APPTAINER_TMPDIR +# temporary directories of the head, the jobs and +# apptainer; on candide REQUIRED, absolute, and not under +# /tmp or /scratch (see tmp_guard) +# +# Example (candide): +# export TMPDIR=/n23data1/$USER/tmp APPTAINER_TMPDIR=/n23data1/$USER/tmp +# workflow/bin/spv run -c ~/runs/grid_3/spv_grid_3.yaml -n +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROFILE="${SPV_PROFILE:-candide}" + +usage() { sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d; s/^# \{0,1\}//'; } + +# --- temporary directories --------------------------------------------------- +# The slurm executor exports this shell's environment to every job, TMPDIR +# literally. On candide the login node's TMPDIR (/scratch/$USER/tmp) cannot be +# created on a compute node, so Slurm falls back to the node's shared 31 GB +# /tmp; inside the container the profile's --cleanenv drops TMPDIR and the host +# /tmp is mounted. Where apptainer cannot mount the image it unpacks all of it +# into its temporary directory -- that filled /tmp on n35 (2026-09-30). Hence: +# on candide both directories must be a path every node sees, outside /tmp and +# /scratch, and the same path goes into the container (snapshot_code). +tmp_guard() { + local v val bad + for v in TMPDIR APPTAINER_TMPDIR; do + val="${!v:-}"; bad="" + if [ -z "$val" ]; then bad="unset" + elif [[ "$val" != /* ]]; then bad="not absolute" + elif [ "$val" = /tmp ] || [[ "$val" == /tmp/* ]]; then bad="under /tmp" + elif [ "$PROFILE" = candide ] && [[ "$val" == /scratch/* ]]; then + bad="under /scratch, which compute nodes cannot create" + fi + if [ -n "$bad" ]; then + if [ "$PROFILE" = candide ]; then + echo "spv: $v is $bad (${val:-}). On candide set TMPDIR and" \ + "APPTAINER_TMPDIR to a directory every node sees, e.g." \ + "/n23data1/\$USER/tmp, before \`spv run\`." >&2 + exit 2 + fi + echo "spv: warning: $v is $bad (${val:-})" >&2 + continue + fi + mkdir -p "$val" + done + [ -z "${APPTAINER_TMPDIR:-}" ] || export SINGULARITY_TMPDIR="$APPTAINER_TMPDIR" +} + +# --- arguments --------------------------------------------------------------- +cmd="${1:-}" +case "$cmd" in ""|-h|--help) usage; exit 0 ;; esac +[ "$cmd" = run ] || [ "$cmd" = status ] && shift || cmd=passthrough + +RUN_CONFIG=""; TARGETS=(); REST=() +while [ $# -gt 0 ]; do + case "$1" in + -c|--config-file) [ $# -ge 2 ] || { echo "spv: $1 needs a file" >&2; exit 2; } + RUN_CONFIG="$2"; shift 2 ;; + -t|--target) [ $# -ge 2 ] || { echo "spv: $1 needs a target" >&2; exit 2; } + TARGETS+=("$2"); shift 2 ;; + --) shift; REST+=("$@"); break ;; + *) REST+=("$1"); shift ;; + esac +done + +# --- status: read-only, no state dir ------------------------------------------- +if [ "$cmd" = status ]; then + STATUS="$REPO/workflow/scripts/status_sims.py" + flags=(); dir="" + for a in ${REST[@]+"${REST[@]}"}; do + if [ -z "$dir" ] && [ -d "$a" ]; then dir="$a"; else flags+=("$a"); fi + done + [ ${#flags[@]} -gt 0 ] || flags=(-S) + if [ -n "$RUN_CONFIG" ]; then + exec python3 "$STATUS" -c "$RUN_CONFIG" "${flags[@]}" + fi + [ -n "$dir" ] || { echo "spv status: give -c or a directory" >&2; exit 2; } + shopt -s nullglob; cfgs=("$dir"/sp_*.yaml) + [ ${#cfgs[@]} -gt 0 ] || { echo "spv status: no sp_*.yaml in $dir" >&2; exit 2; } + rc=0 + for c in "${cfgs[@]}"; do + echo; echo "############## $(basename "$c" .yaml) ##############" + python3 "$STATUS" -c "$c" "${flags[@]}" || rc=$? + done + exit $rc +fi + +# --- run / passthrough: the campaign's state dir -------------------------------- +[ -n "$RUN_CONFIG" ] || { echo "spv: -c is required" >&2; exit 2; } +[ -f "$RUN_CONFIG" ] || { echo "spv: run config $RUN_CONFIG does not exist" >&2; exit 2; } +RUN_CONFIG="$(cd "$(dirname "$RUN_CONFIG")" && pwd)/$(basename "$RUN_CONFIG")" +[ -d "$REPO/workflow/profiles/$PROFILE" ] || { + echo "spv: no profile workflow/profiles/$PROFILE (SPV_PROFILE=$PROFILE)" >&2; exit 2; } +command -v snakemake >/dev/null || { echo "spv: snakemake is not on PATH" >&2; exit 2; } + +GRIDS_BASE="$(python3 - "$REPO/workflow/image_sims/config.yaml" "$RUN_CONFIG" <<'PY' +import sys, yaml +main, user = (yaml.safe_load(open(p)) or {} for p in sys.argv[1:3]) +val = (user.get("image_sims") or {}).get("grids_base") \ + or (main.get("image_sims") or {}).get("grids_base") +print(val or "") +PY +)" +[ -n "$GRIDS_BASE" ] || { echo "spv: image_sims.grids_base is not set in $RUN_CONFIG" >&2; exit 2; } +STATE_DIR="${SPV_STATE_DIR:-$GRIDS_BASE/spv-state}" +SNAPSHOT="$STATE_DIR/code" +tmp_guard + +# --- the launch code snapshot --------------------------------------------------- +# As in ShapePipe's sp: the jobs re-read the Snakefile, rules and scripts when +# they start, so a run executed straight from the checkout picks up whatever is +# edited there meanwhile. spv copies what it launches into $STATE_DIR/code and +# runs entirely out of it; image_sims.sp_validation_repo points there. Refreshed +# wholesale on every `spv run`. +snapshot_code() { + mkdir -p "$SNAPSHOT" + rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ + "$REPO/workflow" "$REPO/src" "$REPO/scripts" "$REPO/config" "$SNAPSHOT/" + cp "$RUN_CONFIG" "$SNAPSHOT/run_config.yaml" + + # Temporary directory inside the container: --cleanenv drops TMPDIR, so the + # head's TMPDIR goes in as a literal --env (tempfile, joblib). It must lie + # under a --bind, or the container could not write it. + python3 - "$SNAPSHOT/workflow/profiles/$PROFILE/config.yaml" "${TMPDIR:-}" <<'PY' +import re, sys, yaml +path, tmp = sys.argv[1], sys.argv[2] +prof = yaml.safe_load(open(path)) +args = prof.get("apptainer-args", "") +if tmp and not tmp.startswith("/tmp"): + roots = [r for b in re.findall(r"--bind (\S+)", args) + for r in (s.split(":")[0] for s in b.split(","))] + if not any(tmp == r or tmp.startswith(r.rstrip("/") + "/") for r in roots): + sys.exit(f"spv: TMPDIR {tmp} is not under any --bind of the profile " + f"({', '.join(roots)}); the container could not write it") + args = re.sub(r"\s*--env (TMPDIR|JOBLIB_TEMP_FOLDER)=\S+", "", args) + prof["apptainer-args"] = f"{args} --env TMPDIR={tmp} --env JOBLIB_TEMP_FOLDER={tmp}" + with open(path, "w") as fh: + fh.write("# Launch snapshot of the profile, rewritten by spv.\n") + yaml.safe_dump(prof, fh, sort_keys=False) + print(f"spv: temporary directory {tmp} (head, jobs and container)", file=sys.stderr) +PY + + python3 - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' +import json, subprocess, sys, time +out, repo = sys.argv[1], sys.argv[2] +def git(*a): + p = subprocess.run(["git", "-C", repo, *a], capture_output=True, text=True) + return p.stdout.strip() if p.returncode == 0 else None +status = git("status", "--porcelain") +json.dump({"_comment": "Code snapshot taken by `spv run`; the run executes " + "this tree, never the checkout.", + "source": repo, "taken_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "head": git("rev-parse", "HEAD"), + "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(status), + "dirty_files": status.splitlines() if status else []}, + open(out, "w"), indent=2) +PY + echo "spv: code snapshot refreshed at $SNAPSHOT" >&2 +} + +sm() { + [ -f "$SNAPSHOT/workflow/image_sims/Snakefile" ] || { + echo "spv: no snapshot at $SNAPSHOT yet; run \`spv run\` first" >&2; exit 2; } + cd "$STATE_DIR" + # Targets and the caller's arguments BEFORE --configfile/--config: both take + # one-or-more values and would swallow a target that follows them. + snakemake ${TARGETS[@]+"${TARGETS[@]}"} ${REST[@]+"${REST[@]}"} \ + -s "$SNAPSHOT/workflow/image_sims/Snakefile" \ + --profile "$SNAPSHOT/workflow/profiles/$PROFILE" \ + --directory "$STATE_DIR" \ + --configfile "$SNAPSHOT/run_config.yaml" \ + --config "image_sims={\"sp_validation_repo\": \"$SNAPSHOT\"}" +} + +if [ "$cmd" = run ]; then + snapshot_code + [ ${#TARGETS[@]} -gt 0 ] || TARGETS=(im_mbias) +fi +sm diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile index 7420ae68..0a95da64 100644 --- a/workflow/image_sims/Snakefile +++ b/workflow/image_sims/Snakefile @@ -4,20 +4,15 @@ Run the sp_validation-side chain (merge -> extract -> calibrate -> m-bias), optionally including the ShapePipe pipeline stage, without pulling in the cosmology-validation config the top-level ``workflow/Snakefile`` requires. -The one drive command on candide, with the committed SLURM profile owning -scheduling and container wrapping: +Launch it with ``workflow/bin/spv run -c my_run.yaml`` (``-n`` for a dry run), +which snapshots the code, sets the temporary directories and passes the profile; +see ``spv --help``. ``configfile:`` below loads the operational defaults (the +main config), so ``my_run.yaml`` need only carry the science knobs and any +override; Snakemake deep-merges the two. - snakemake --profile workflow/profiles/candide \\ - -s workflow/image_sims/Snakefile \\ - im_mbias --configfile my_run.yaml - -``configfile:`` below loads the operational defaults, so ``my_run.yaml`` need -only carry the science knobs (and any operational override); Snakemake -deep-merges the two. Off-cluster, drop ``--profile`` and add ``-j N``. - -Put targets *before* ``--configfile``: it takes one-or-more paths, so a target -after it is swallowed as a config path ("No such file: im_mbias"). Always -dry-run first with ``-n``. +Driving snakemake directly also works: put targets *before* ``--configfile`` +(it takes one-or-more paths, so a target after it is swallowed as a config +path), and on candide set TMPDIR/APPTAINER_TMPDIR yourself. The same rules are also included in the main workflow under ``if "image_sims" in config``. @@ -34,7 +29,9 @@ sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), ".."))) import common -configfile: "workflow/image_sims/config.yaml" +# Next to this file, not relative to the working directory: spv runs snakemake +# from the campaign's state directory, out of a code snapshot. +configfile: os.path.join(str(workflow.basedir), "config.yaml") include: "../rules/image_sims.smk" diff --git a/workflow/scripts/im_mbias_config.py b/workflow/scripts/im_mbias_config.py index cf6fdba2..9abfba98 100644 --- a/workflow/scripts/im_mbias_config.py +++ b/workflow/scripts/im_mbias_config.py @@ -35,6 +35,20 @@ def _git(repo, *args): return None +def _repo_provenance(repo): + """Branch and commit of ``repo``. A code snapshot taken by ``spv run`` or + ``sp run`` has no .git; its ``snapshot.json`` records the checkout it was + copied from, including whether that checkout had uncommitted changes.""" + snap = os.path.join(repo, "snapshot.json") + if os.path.isfile(snap): + with open(snap) as fh: + info = yaml.safe_load(fh) or {} + return {"branch": info.get("branch"), "commit": info.get("head"), + "dirty": info.get("dirty"), "snapshot_of": info.get("source")} + return {"branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _git(repo, "rev-parse", "HEAD")} + + def _sif_revision(): """``org.opencontainers.image.revision`` from the running image's OCI labels. @@ -84,16 +98,8 @@ def _sif_revision(): "output_path": params.results, "provenance": { "manifest_sha256": manifest_sha256, - "sp_validation": { - "branch": _git( - params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD" - ), - "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), - }, - "shapepipe": { - "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), - }, + "sp_validation": _repo_provenance(params.sp_validation_repo), + "shapepipe": _repo_provenance(params.shapepipe_repo), "container": { "sif": params.sif, "ghcr_revision": _sif_revision(), diff --git a/workflow/scripts/status_sims.py b/workflow/scripts/status_sims.py new file mode 100644 index 00000000..c9ff7f5c --- /dev/null +++ b/workflow/scripts/status_sims.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Disk-scan status report for image-sims ShapePipe campaigns. + +Not a DAG node (same reasoning as shapepipe/workflow/scripts/run_report.py: +a report that's part of the DAG would itself be poisoned by the failures it +must enumerate). Run any time, mid-run or after: + + workflow/bin/spv status -c sp_1p2z_grid_3.yaml # one campaign (sp run config) + workflow/bin/spv status # every sp_*.yaml in it + python workflow/scripts/status_sims.py # directly + +For each tile/exposure and each stage, a manifest means complete, a log +with no manifest means failed, and neither means not yet attempted -- the +same three-way read completeness.py's own contract establishes. + +A failed stage's log also carries why it failed: the runner counts, and the +lines completeness_sims.py scraped out of the shapepipe logs. Those are +reported per unit and grouped by reason, so one bad input among hundreds of +tiles is readable without opening any log by hand. +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +# Two layouts share this script. The NATIVE sp_validation prototype fits the +# PSF at tile level (tile_fake_psf) and runs ngmix as one call; the UNIFIED +# ShapePipe workflow (shapepipe #891) moves the PSF to the exposure level +# (exp_psf), chunks ngmix into tile_ngmix_1..N and merges the chunks +# (tile_merge_cats). Pass --layout to force one; `auto` reads what is on disk. +NATIVE_TILE_STAGES = [ + "tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_fake_psf", "tile_vignets", + "tile_ngmix", "tile_make_cat", +] +NATIVE_EXP_STAGES = ["exp_get_images", "exp_split"] + +UNIFIED_TILE_STAGES = [ + "tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat", +] +UNIFIED_EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] + +# Back-compat for anything importing these names. +TILE_STAGES = NATIVE_TILE_STAGES +EXP_STAGES = NATIVE_EXP_STAGES + +# The Gen-2 RUN_NAME suffixes (run_sp_tile_Git, run_sp_exp_Gie, ...) -- +# shorter and more familiar than the stage names for a wide table. +STAGE_ABBR = { + "tile_get_images": "Git", + "tile_uncompress": "Uz", + "tile_find_exposures": "Fe", + "tile_merge_headers": "Mh_exp", + "tile_detect": "Sx", + "tile_fake_psf": "fpsf", + "tile_vignets": "ViVi", + "tile_ngmix": "Ng", + "tile_merge_cats": "Ms", + "tile_make_cat": "Mc", + "exp_get_images": "Gie", + "exp_split": "Sp", + "exp_psf": "Psf", +} + +SYMBOL = {"complete": ".", "failed": "X", "missing": " "} + + +_CHUNK_RE = re.compile(r"^(tile_ngmix)_\d+$") + + +def unit_records(unit_dir: Path): + """(done, failed) stage-name sets for one unit. + + A reclaimed unit has had its manifests deleted, so clean_tile/ + clean_exposure's tombstone is read as the record of what completed -- it + absorbs every manifest verbatim for exactly this reason. Without this a + FINISHED campaign reports all zeros, because the more complete a unit is + the fewer manifests survive on disk. + """ + done, failed = set(), set() + mdir, ldir = unit_dir / "manifests", unit_dir / "logs" + if mdir.is_dir(): + done |= {p.stem for p in mdir.glob("*.json")} + if ldir.is_dir(): + # An EMPTY log means the job never ran, not that it failed: snakemake + # creates the declared log path even for a job its cone never reached. + # Counting those as failures turned one dead tile into a column of Xs + # across every downstream stage (run_report calls them "not run"). + failed |= {p.stem for p in ldir.glob("*.json") + if p.stat().st_size > 0} + tomb = unit_dir / "cleaned.json" + if tomb.exists(): + try: + body = json.loads(tomb.read_text()) + except (OSError, ValueError): + body = {} + done |= set((body.get("manifests") or {})) + return done, failed - done + + +def unit_statuses(unit_dir: Path, stages: list, n_chunks: int) -> dict: + """stage -> complete | failed | missing, with the chunked ngmix collapsed + into its single `tile_ngmix` column (complete only when every chunk is).""" + done, failed = unit_records(unit_dir) + + chunks_done = {m for m in done if _CHUNK_RE.match(m)} + chunks_failed = {m for m in failed if _CHUNK_RE.match(m)} + + out = {} + for s in stages: + if s == "tile_ngmix" and n_chunks > 1: + if len(chunks_done) >= n_chunks: + out[s] = "complete" + elif chunks_failed or chunks_done: + out[s] = "failed" if chunks_failed else "missing" + else: + out[s] = "missing" + elif s in done: + out[s] = "complete" + elif s in failed: + out[s] = "failed" + else: + out[s] = "missing" + return out + + +def units_of(run_dir: Path, kind: str) -> list: + root = run_dir / kind + return sorted(p for p in root.glob("*/*") if p.is_dir()) if root.is_dir() else [] + + +def ngmix_chunks(run_dir: Path) -> int: + """How many chunks this run splits ngmix into: the most any one tile + records. 1 means the native single-call layout.""" + best = 1 + for unit_dir in units_of(run_dir, "tiles"): + done, failed = unit_records(unit_dir) + n = len({m for m in (done | failed) if _CHUNK_RE.match(m)}) + best = max(best, n) + return best + + +def detect_layout(run_dir: Path) -> str: + """'unified' or 'native'. + + Structural markers first, because they exist from the first minute of a + run: the native prototype keeps a plain JSON index inside the run dir, + while bin/sp puts its snakemake state in a `-state` sibling and + its index in a sqlite file under the products root. Stage-name sniffing is + the fallback, and on its own it mis-reads a young unified run as native -- + none of tile_ngmix_*, tile_merge_cats or exp_psf has been reached yet. + """ + if (run_dir / "index.json").is_file(): + return "native" + if run_dir.with_name(run_dir.name + "-state").is_dir(): + return "unified" + for name in ("product", "products"): + if (run_dir.parent / name / "index" / "run_index.sqlite").is_file(): + return "unified" + + for kind in ("tiles", "exp"): + for unit_dir in units_of(run_dir, kind): + done, failed = unit_records(unit_dir) + names = done | failed + if any(_CHUNK_RE.match(m) for m in names) or "tile_merge_cats" in names \ + or "exp_psf" in names: + return "unified" + if "tile_fake_psf" in names: + return "native" + return "native" + + +def stage_status(unit_dir: Path, stage: str) -> str: + """Kept for callers that ask about one stage; prefer unit_statuses.""" + return unit_statuses(unit_dir, [stage], 1)[stage] + + +# Group reasons that differ only in tile id, path or count, so N tiles failing +# the same way collapse to one line. +# Two segments minimum, so a count like "0/3" is not mistaken for a path. +_PATH_RE = re.compile(r"(?:/[^/\s]+){2,}") +_NUM_RE = re.compile(r"\d+") + + +def reason_shape(reason: str) -> str: + return _NUM_RE.sub("N", _PATH_RE.sub("", reason)) + + +def short_paths(reason: str) -> str: + """Absolute paths down to their file name: every run dir prefix is the + same, and the full lines are far too wide to scan.""" + return _PATH_RE.sub(lambda m: m.group(0).rsplit("/", 1)[-1], reason) + + +def stage_failures(unit_dir: Path, stage: str, n_chunks: int = 1) -> list: + """Why `stage` failed, as " found/expect" plus the scraped lines. + + Reads the log completeness_sims.py wrote. An unreadable or reason-less log + still counts as a failure -- it just cannot say more than that. + """ + logs = [unit_dir / "logs" / f"{stage}.json"] + if stage == "tile_ngmix" and n_chunks > 1: + # No tile_ngmix.json exists in the chunked layout; each chunk logs + # separately, and any of them can be the one that failed. + logs = sorted((unit_dir / "logs").glob("tile_ngmix_*.json")) + + out = [] + for log in logs: + try: + body = json.loads(log.read_text()) + except (OSError, ValueError): + out.append(f"{log.name}: log unreadable") + continue + tag = f"{log.stem}: " if len(logs) > 1 else "" + for rec in body.get("failures", []): + head = f"{tag}{rec.get('runner')}: {rec.get('found')}/{rec.get('expect')}" + reasons = rec.get("reasons") or [] + out.append(head if not reasons else f"{head} -- {reasons[0]}") + out += reasons[1:] + return out or ["no reason recorded"] + + +def report_failures(run_dir: Path, kind: str, stages: list, + n_chunks: int = 1) -> None: + """Every failed unit with its reasons, then the same grouped by reason.""" + units = units_of(run_dir, kind) + + failed, groups = [], {} + for unit_dir in units: + statuses = unit_statuses(unit_dir, stages, n_chunks) + for s in stages: + if statuses[s] != "failed": + continue + reasons = stage_failures(unit_dir, s, n_chunks) + failed.append((unit_dir.name, s, reasons)) + for r in reasons: + key = (s, reason_shape(r)) + groups.setdefault(key, [r, []])[1].append(unit_dir.name) + + if not failed: + print(f"no failed {kind}") + print() + return + + n_units = len({u for u, _, _ in failed}) + print(f"{n_units} {kind} failed, {len(failed)} stage failures") + print() + for unit, stage, reasons in failed: + print(f" {unit:<12} {STAGE_ABBR[stage]}") + for r in reasons: + print(f" {short_paths(r)}") + print() + + # Only worth a second pass when a reason actually spans several units. + shared = {k: v for k, v in groups.items() if len(v[1]) > 1} + if not shared: + return + print(" by reason:") + for (stage, _), (reason, hit) in sorted(shared.items(), + key=lambda kv: -len(kv[1][1])): + print(f" {len(hit):>4}x {STAGE_ABBR[stage]:<8} {short_paths(reason)}") + print(f" {', '.join(hit[:8])}" + (" ..." if len(hit) > 8 else "")) + print() + + +def report_finished(run_dir: Path, kind: str, stages: list, + n_chunks: int = 1) -> None: + """--very-short: the abbreviated header, and ONE row below it -- how many + units have COMPLETED each step -- instead of one row per unit.""" + units = units_of(run_dir, kind) + n_cleaned = sum(1 for u in units if (u / "cleaned.json").exists()) + print(f"{len(units)} {kind}") + if not units: + return + if n_cleaned: + # Say it outright. The counts below come from the tombstones once a + # unit is reclaimed, so a FINISHED, fully cleaned campaign prints a row + # of full numbers and looks untouched -- the opposite of the pre- + # tombstone behaviour, where it printed zeros and looked emptied. + what = "store deleted" if n_cleaned == len(units) else "stores deleted" + print(f" {n_cleaned}/{len(units)} RECLAIMED ({what}); the row below is " + f"what ran, read back from cleaned.json") + + header = f"{'':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) + print(header) + print("-" * len(header)) + + counts = {s: 0 for s in stages} + for unit_dir in units: + statuses = unit_statuses(unit_dir, stages, n_chunks) + for s in stages: + if statuses[s] == "complete": + counts[s] += 1 + row = f"{'complete':<12}" + "".join(f"{counts[s]:>8}" for s in stages) + print(row) + + +def report_units(run_dir: Path, kind: str, stages: list, *, short: bool = False, + n_chunks: int = 1) -> None: + units = units_of(run_dir, kind) + if not units: + print(f"No {kind} units found under {run_dir / kind}") + return + + counts = {s: {"complete": 0, "failed": 0, "missing": 0} for s in stages} + + header = f"{'unit':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) + print(header) + print("-" * len(header)) + + for unit_dir in units: + unit = unit_dir.name + row = f"{unit:<12}" + statuses = unit_statuses(unit_dir, stages, n_chunks) + for s in stages: + st = statuses[s] + counts[s][st] += 1 + row += f"{SYMBOL[st]:>8}" + print(row) + + # --short: just the per-unit table above, skip the aggregate counts below. + if short: + print() + return + + print() + print(f"{len(units)} {kind} total") + for s in stages: + c = counts[s] + print(f" {STAGE_ABBR[s]:<8} complete={c['complete']:<4} failed={c['failed']:<4} " + f"not-yet-run={c['missing']}") + print() + + +def final_cats(products_dir: Path) -> list: + """The published catalogues, on the products root (which is run_dir when + the run config sets no products_dir).""" + return sorted(products_dir.glob("tiles/*/*/final_cat-*.fits")) + + +def guess_products(run_dir: Path) -> Path: + """Where the catalogues are when only a run dir was given. + + The native layout publishes under the run root itself; the unified one + puts run/ and product/ side by side, so a bare `status_sims.py ` + would otherwise report `final_cat produced: 0` for a finished campaign. + """ + if final_cats(run_dir): + return run_dir + for name in ("product", "products"): + cand = run_dir.parent / name + if final_cats(cand): + return cand + return run_dir + + +def report_disk(run_dir: Path, enabled: bool) -> None: + """What the run directory still occupies -- the direct answer to "was this + cleaned up?", which the stage table alone cannot give.""" + if not enabled: + return + total = files = 0 + for dirpath, _, names in os.walk(run_dir): + for n in names: + try: + total += os.stat(os.path.join(dirpath, n)).st_size + files += 1 + except OSError: + pass + for unit, div in (("T", 1 << 40), ("G", 1 << 30), ("M", 1 << 20), ("K", 1 << 10)): + if total >= div: + size = f"{total / div:.1f}{unit}" + break + else: + size = f"{total}B" + print(f"run dir on disk: {size} in {files} files") + + +def report_merged(products_dir: Path, cats: list) -> None: + """The merged hdf5, the campaign's hand-off to sp_validation. + + Per-tile catalogues are only half the story: the sp_validation side starts + from ONE hdf5 per branch, so a campaign whose tiles are all green but whose + merge never ran is not actually finished. Checked against the per-tile + FITS, because a merge that silently picked up a subset is the failure worth + catching -- create_final_cat.py skips a tile it cannot find and says so + only in its own verbose output. + """ + merged = sorted(products_dir.glob("final_cat_*.hdf5")) + if not merged: + print(f"merged hdf5: none in {products_dir} " + f"(run merge_final_cats, or create_final_cat.py -I -c )") + return + + for path in merged: + try: + import h5py + except ImportError: + print(f"merged hdf5: {path.name} ({path.stat().st_size / 1e6:.0f} MB)" + " -- install h5py to check its contents") + return + try: + with h5py.File(path, "r") as fh: + groups = {} + def walk(name, obj): + if isinstance(obj, h5py.Dataset): + groups[name] = obj.shape[0] + fh.visititems(walk) + except OSError as exc: + print(f"merged hdf5: {path.name} UNREADABLE ({exc})") + continue + + n_tiles, n_rows = len(groups), sum(groups.values()) + # Row totals from the per-tile FITS headers only -- no pixel data read. + src_rows = 0 + for c in cats: + try: + from astropy.io import fits + with fits.open(c, memmap=True) as hdus: + src_rows += hdus[1].header["NAXIS2"] + except Exception: + src_rows = -1 + break + + size = path.stat().st_size / 1e6 + note = "" + if len(cats) and n_tiles != len(cats): + note = f" MISMATCH: {len(cats)} final_cat files on disk" + elif src_rows >= 0 and n_rows != src_rows: + note = f" MISMATCH: {src_rows:,} rows in the per-tile catalogues" + print(f"merged hdf5: {path.name} {n_tiles} tiles, {n_rows:,} rows, " + f"{size:.0f} MB{note}") + + n_file = products_dir / "n_tiles_final.txt" + if n_file.exists(): + recorded = n_file.read_text().strip() + flag = "" if recorded == str(len(cats)) else f" MISMATCH: {len(cats)} on disk" + print(f"n_tiles_final.txt: {recorded}{flag}") + + +def _expand(config: dict) -> dict: + """Expand $name and ${name} in every value, from the top-level scalar keys, + repeatedly (as ShapePipe's run_config.py does for an sp run config).""" + scalars = {k: v for k, v in config.items() + if isinstance(v, (str, int, float)) and not isinstance(v, bool)} + + def sub(value, depth=0): + if isinstance(value, str): + out = re.sub(r"\$\{(\w+)\}|\$(\w+)", + lambda m: str(scalars.get(m.group(1) or m.group(2), + m.group(0))), value) + return out if out == value or depth >= 5 else sub(out, depth + 1) + if isinstance(value, dict): + return {k: sub(v, depth) for k, v in value.items()} + if isinstance(value, list): + return [sub(v, depth) for v in value] + return value + + return {k: sub(v) for k, v in config.items()} + + +def run_dirs_from_config(path: Path): + """(run_dir, products_dir) from a run config's outputs: block; products_dir + defaults to run_dir, as in ShapePipe's workflow.""" + import yaml + + config = _expand(yaml.safe_load(Path(path).read_text()) or {}) + outputs = config.get("outputs") or {} + run_dir = outputs.get("run_dir") + if not run_dir or "$" in str(run_dir): + raise ValueError(f"{path}: outputs.run_dir is unset or unexpanded " + f"({run_dir!r})") + return Path(run_dir), Path(outputs.get("products_dir") or run_dir) + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("run_dir", type=Path, nargs="?", + help="run directory; omit when passing -c") + p.add_argument("-c", "--config", type=Path, + help="ShapePipe run config (sp_*.yaml): read outputs.run_dir " + "and outputs.products_dir from it, $name/${name} expanded") + p.add_argument("-s", "--short", action="store_true", + help="per-tile/per-exposure table only, no aggregate " + "'N units total' / per-stage count summary") + p.add_argument("-S", "--very-short", action="store_true", + help="no per-unit table at all -- just 'N/M finished' " + "for tiles and exposures, once each (overrides -s)") + p.add_argument("-p", "--products", type=Path, + help="products root, when it is neither the run dir nor a " + "product/ sibling of it") + p.add_argument("--du", action="store_true", + help="also measure the run directory on disk (a full walk; " + "slow on a live campaign, instant on a reclaimed one)") + p.add_argument("--layout", choices=("auto", "native", "unified"), + default="auto", + help="stage set to report: the sp_validation native " + "prototype, the unified ShapePipe workflow, or auto " + "(default) to read it off the run dir") + args = p.parse_args(argv) + + if args.config: + try: + run_dir, products_dir = run_dirs_from_config(args.config) + except (OSError, ValueError) as exc: + p.error(f"{exc}") + elif args.run_dir: + run_dir = args.run_dir + products_dir = guess_products(run_dir) + else: + p.error("give a run directory, or -c with a run config") + + if args.products: + products_dir = args.products + + layout = args.layout if args.layout != "auto" else detect_layout(run_dir) + tile_stages = UNIFIED_TILE_STAGES if layout == "unified" else NATIVE_TILE_STAGES + exp_stages = UNIFIED_EXP_STAGES if layout == "unified" else NATIVE_EXP_STAGES + n_chunks = ngmix_chunks(run_dir) if layout == "unified" else 1 + + print(f"=== {run_dir} ===") + if products_dir != run_dir: + print(f"=== products: {products_dir} ===") + chunk_note = f", ngmix x{n_chunks}" if n_chunks > 1 else "" + print(f"=== layout: {layout}{chunk_note} ===") + + if args.very_short: + report_finished(run_dir, "tiles", tile_stages, n_chunks) + report_finished(run_dir, "exp", exp_stages) + cats = final_cats(products_dir) + print(f"final_cat produced: {len(cats)}") + report_merged(products_dir, cats) + report_disk(run_dir, args.du) + return 0 + + print() + print("--- tiles ---") + report_units(run_dir, "tiles", tile_stages, short=args.short, n_chunks=n_chunks) + print("--- exposures ---") + report_units(run_dir, "exp", exp_stages, short=args.short) + + if not args.short: + print("--- failures ---") + report_failures(run_dir, "tiles", tile_stages, n_chunks) + report_failures(run_dir, "exp", exp_stages) + + report_disk(run_dir, args.du) + cats = final_cats(products_dir) + print(f"final_cat produced: {len(cats)}") + if not args.short: + for f in cats: + print(f" {f}") + report_merged(products_dir, cats) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From b268fd64e387397ee511dfa8ce0dcffb8e191f30 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 2 Oct 2026 10:20:11 +0200 Subject: [PATCH 20/36] image sims: bring over the uncommitted work of imsims_workflow The state the grid m-bias runs actually used (sp_validation_imsims_workflow working tree, 2026-10-02), so that spv runs it: the image_sims.smk chain on top of ShapePipe's own workflow (init -> extract -> calibrate -> m-bias), the matching workflow/image_sims/config.yaml, the per-tile response jackknife and related calibration/m-bias changes with their tests, and the mask configs mask_v1.X.10_im_sim{,_R_now}.yaml. Tests: test_calibration.py and test_image_sims.py pass (20) in the m-bias image. Left out: the profile's `config: image_sims={"sif": ...}` -- the image is a per-user choice and goes in the run config (image_sims.sif). Added: n35 to the candide excludes (its /tmp was filled on 2026-09-30). Co-Authored-By: Claude Opus 5.5 --- config/calibration/mask_v1.X.10_im_sim.yaml | 75 ++++++ .../mask_v1.X.10_im_sim_R_now.yaml | 75 ++++++ .../calibrate_comprehensive_cat.py | 68 ++++++ scripts/compute_m_bias_image_sims.py | 47 +++- src/sp_validation/calibration.py | 230 ++++++++++++++++-- src/sp_validation/image_sims.py | 216 ++++++++++++++-- src/sp_validation/tests/test_calibration.py | 81 ++++++ src/sp_validation/tests/test_image_sims.py | 219 +++++++++++++++++ workflow/image_sims/config.yaml | 60 ++--- workflow/profiles/candide/config.yaml | 2 +- workflow/rules/image_sims.smk | 140 +++++++---- 11 files changed, 1081 insertions(+), 132 deletions(-) create mode 100644 config/calibration/mask_v1.X.10_im_sim.yaml create mode 100644 config/calibration/mask_v1.X.10_im_sim_R_now.yaml diff --git a/config/calibration/mask_v1.X.10_im_sim.yaml b/config/calibration/mask_v1.X.10_im_sim.yaml new file mode 100644 index 00000000..f159fa4c --- /dev/null +++ b/config/calibration/mask_v1.X.10_im_sim.yaml @@ -0,0 +1,75 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, including deblended objects, type v1.X.10 + +# General parameters (can also given on command line) +params: + input_path: shape_catalog_comprehensive_ngmix.fits + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 2 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MCAL_TYPES_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: w + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/config/calibration/mask_v1.X.10_im_sim_R_now.yaml b/config/calibration/mask_v1.X.10_im_sim_R_now.yaml new file mode 100644 index 00000000..3244450f --- /dev/null +++ b/config/calibration/mask_v1.X.10_im_sim_R_now.yaml @@ -0,0 +1,75 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, including deblended objects, type v1.X.10 + +# General parameters (can also given on command line) +params: + input_path: shape_catalog_comprehensive_ngmix.fits + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 2 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MCAL_TYPES_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: null + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 2605338f..40191e69 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -214,6 +214,23 @@ add_cols_data["e1_leak_corrected"] = e1_leak_corrected add_cols_data["e2_leak_corrected"] = e2_leak_corrected +# %% +# Tile IDs and leave-one-tile-out response, for a jackknife over tiles +# downstream (image-sims m-bias): each jackknife sample is recalibrated with +# its own R = R_shear + R_selection, and its own R_joint +add_cols_format = {} +tile_ids_jk = None +if "TILE_ID" in dat.dtype.names: + tile_col = np.asarray(dat["TILE_ID"]).astype(str) + add_cols_data["TILE_ID"] = tile_col[mask_combined._mask][mask_metacal] + add_cols_format["TILE_ID"] = f"{max(len(t) for t in tile_col)}A" + tile_ids_jk, R_jk, R_joint_jk = gal_metacal.response_jackknife( + tile_col[mask_combined._mask] + ) + print(f"Leave-one-tile-out response computed for {len(tile_ids_jk)} tiles") +else: + print("No TILE_ID column, skipping leave-one-tile-out response") + # %% # Add information to FITS header @@ -223,6 +240,32 @@ # Add general and metacal config information to FITS header obj.add_params_to_FITS_header(header, cm=cm) +# %% +# Joint shear and selection response, alternative to R (image-sims m-bias) +for _i in (1, 2): + for _j in (1, 2): + header[f"RJ_{_i}{_j}"] = ( + float(gal_metacal.R_joint[_i - 1][_j - 1]), + f"Joint shear+selection response comp {_i} {_j}", + ) +print(f"Joint response R_joint =\n{gal_metacal.R_joint}\nR =\n{gal_metacal.R}") + +# %% +# Resampling error on the selection response. R_selection divides a tiny +# difference of means by 2h = 0.02, so it is noise-amplifying by construction; +# without this number an R_S of a few percent cannot be told from zero. +for _i in (1, 2): + for _j in (1, 2): + header[f"R_S{_i}{_j}_E"] = ( + float(gal_metacal.R_selection_std[_i - 1][_j - 1]), + f"Resampling std of selection response R_S{_i}{_j}", + ) +print( + "Selection response R_S11 = " + f"{gal_metacal.R_selection[0][0]:+.4f} +- {gal_metacal.R_selection_std[0][0]:.4f}, " + f"R_S22 = {gal_metacal.R_selection[1][1]:+.4f} +- {gal_metacal.R_selection_std[1][1]:.4f}" +) + # %% # Add mask information to FITS header for my_mask in masks: @@ -249,9 +292,34 @@ c_err=c_err, w_type="des", add_cols=add_cols_data, + add_cols_format=add_cols_format, add_header=header, ) +# %% +# Leave-one-tile-out response as extra HDU "R_JK" +if tile_ids_jk is not None: + jk_cols = [ + fits.Column( + name="TILE_ID", array=tile_ids_jk, format=add_cols_format["TILE_ID"] + ) + ] + for key, resp in (("R", R_jk), ("RJ", R_joint_jk)): + for _i in (1, 2): + for _j in (1, 2): + jk_cols.append( + fits.Column( + name=f"{key}_{_i}{_j}", + array=resp[:, _i - 1, _j - 1], + format="D", + ) + ) + jk_hdu = fits.BinTableHDU.from_columns(jk_cols, name="R_JK") + jk_hdu.header["COMMENT"] = "Responses with tile TILE_ID left out:" + jk_hdu.header["COMMENT"] = "R_ij = R_shear + R_select, RJ_ij = joint response" + with fits.open(output_shape_cat_path, mode="append") as hdul: + hdul.append(jk_hdu) + # %% # Write mask information to ASCII file with open("masks.txt", "w") as f_out: diff --git a/scripts/compute_m_bias_image_sims.py b/scripts/compute_m_bias_image_sims.py index 04ca49b7..84f95b92 100644 --- a/scripts/compute_m_bias_image_sims.py +++ b/scripts/compute_m_bias_image_sims.py @@ -275,17 +275,25 @@ def main(): # scalars serialise as !!python/object binary). results = to_python(mb.run(verbose=True)) + # Response methods: R = R_shear + R_select, and R_joint if available + methods = {"R_shear + R_select": results["weights"]} + if "joint" in results: + methods["joint"] = results["joint"]["weights"] + print() print("=" * 40) print(" Results") print("=" * 40) - for scheme, res in results["weights"].items(): - print(f" weights: {scheme}") - print(f" m1 = {res['m1']:+.4f} +-{res['m1_err']:.4f}") - print(f" c1 = {res['c1']:+.4f} +-{res['c1_err']:.4f}") - print(f" m2 = {res['m2']:+.4f} +-{res['m2_err']:.4f}") - print(f" c2 = {res['c2']:+.4f} +-{res['c2_err']:.4f}") - print("=" * 40) + for method, weights in methods.items(): + print(f" response: {method}") + for scheme, res in weights.items(): + print(f" weights: {scheme}") + for key in ("m1", "c1", "m2", "c2"): + line = f" {key} = {res[key]:+.4f} +-{res[key + '_err']:.4f}" + if key + "_err_jk" in res: + line += f" (tile jackknife +-{res[key + '_err_jk']:.4f})" + print(line) + print("=" * 40) # Cumulative tracking if args.cumulative: @@ -326,16 +334,29 @@ def main(): with open(txt_path, "w") as f: f.write("Multiplicative and additive shear bias from image simulations\n") f.write("=" * 60 + "\n") - for scheme, res in results["weights"].items(): - f.write(f"\nweights: {scheme}\n") - f.write(f" m1 = {res['m1']:+.6f} ± {res['m1_err']:.6f}\n") - f.write(f" c1 = {res['c1']:+.6f} ± {res['c1_err']:.6f}\n") - f.write(f" m2 = {res['m2']:+.6f} ± {res['m2_err']:.6f}\n") - f.write(f" c2 = {res['c2']:+.6f} ± {res['c2_err']:.6f}\n") + for method, weights in methods.items(): + f.write(f"\nresponse: {method}\n") + for scheme, res in weights.items(): + f.write(f"\nweights: {scheme}\n") + for key in ("m1", "c1", "m2", "c2"): + line = f" {key} = {res[key]:+.6f} ± {res[key + '_err']:.6f}" + if key + "_err_jk" in res: + line += f" (tile jackknife ± {res[key + '_err_jk']:.6f})" + f.write(line + "\n") f.write( "\nErrors computed via bootstrap resampling " f"(n={config['n_bootstrap']} resamples)\n" ) + if any("m1_err_jk" in res for res in results["weights"].values()): + f.write( + "Tile jackknife: leave one tile out, recalibrated with the" + " response computed without that tile\n" + ) + if "joint" in methods: + f.write( + "Response joint: R_ij = ( over sel_jp - over sel_jm)" + " / 2h, g and selection from the same sheared branch\n" + ) print(f"Results written to {txt_path}") if args.cumulative: diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index 9eaaf42e..1e0cfe1c 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -903,7 +903,9 @@ def _compute_calibration(self): self._shear_response() self._selection_response() + self._selection_response_std() self._total_response() + self._joint_response() # self._shear_response_std(stat_operator=lambda x: # jackknif_weighted_average(x, np.ones_like(x))) @@ -1068,11 +1070,42 @@ def _shear_response_std( [[self.R11_stds, self.R12_stds], [self.R21_stds, self.R22_stds]] ) - def _selection_response(self): - """Add docstring. + def _mean_ns(self, comp, mask, data=None): + """Mean of a component over ``mask``, weighted like the total. - ... + The component is taken from the noshear branch, or from ``data`` (a + sheared branch) if given. Weighted with ``global_R_weight`` when it is + set, unweighted otherwise -- the same switch ``_total_response`` + applies to the shear term. The weights are always the noshear ones. + """ + values = (self.ns if data is None else data)[comp][mask] + if self._global_R_weight is None or self._global_R_weight == "None": + return np.mean(values) + return np.average(values, weights=self.ns[self._global_R_weight][mask]) + def _selection_response(self): + """Response of the SELECTION, as a difference of two sample means. + + Unlike the shear response there is no per-object value here: the two + means are taken over DIFFERENT subsets -- the noshear ellipticities + selected by the +/- sheared catalogues' own masks -- so this quantity + exists only at ensemble level and cannot be binned, nor averaged over + objects. + + It can still carry weights, and must. The estimator divides a weighted + mean ellipticity by R = R_shear + R_selection, so both terms have to be + the response OF THAT WEIGHTED MEAN. For the shear term that means a + weighted mean of per-object responses; for this one it means weighted + means INSIDE the difference. Leaving it unweighted while the shear term + is weighted leaves R inconsistent with the average it calibrates, at + the few-percent level R_selection/R reaches in practice. + + NOT included: the response of the WEIGHTS themselves. ``w_des`` is a + function of snr and size_ratio, both of which shift under metacal + shear, so a complete treatment recomputes the weights on each sheared + counterpart and folds that term in here (Gatti et al. 2021). That needs + weights available per sheared catalogue, which they are not -- w_des is + derived from the response, after calibration. """ ma_p1 = self.mask_dict["p1"] ma_m1 = self.mask_dict["m1"] @@ -1080,23 +1113,190 @@ def _selection_response(self): ma_m2 = self.mask_dict["m2"] h2 = 2 * self._step - self.R11_s = ( - np.mean(self.ns["g1"][ma_p1]) - np.mean(self.ns["g1"][ma_m1]) - ) / h2 - self.R22_s = ( - np.mean(self.ns["g2"][ma_p2]) - np.mean(self.ns["g2"][ma_m2]) - ) / h2 - self.R12_s = ( - np.mean(self.ns["g1"][ma_p2]) - np.mean(self.ns["g1"][ma_m2]) - ) / h2 - self.R21_s = ( - np.mean(self.ns["g2"][ma_p1]) - np.mean(self.ns["g2"][ma_m1]) - ) / h2 + self.R11_s = (self._mean_ns("g1", ma_p1) - self._mean_ns("g1", ma_m1)) / h2 + self.R22_s = (self._mean_ns("g2", ma_p2) - self._mean_ns("g2", ma_m2)) / h2 + self.R12_s = (self._mean_ns("g1", ma_p2) - self._mean_ns("g1", ma_m2)) / h2 + self.R21_s = (self._mean_ns("g2", ma_p1) - self._mean_ns("g2", ma_m1)) / h2 self.R_selection = np.array( [[self.R11_s, self.R12_s], [self.R21_s, self.R22_s]] ) + def _response_from_selections(self, sel, joint=False): + """Response as a difference of means over the sheared selections. + + R_ij = ( over sel_jp - over sel_jm) / 2h. With + ``joint=False`` g is the noshear ellipticity, which gives the + selection response R_selection. With ``joint=True`` g is measured on + the sheared branch itself, which gives shear and selection response + in one go, R_joint ~ R_shear + R_selection (metadetection, Sheldon et + al. 2020). + + Parameters + ---------- + sel : dict + index arrays for the keys "p1", "m1", "p2", "m2" + joint : bool, optional + see above; default is ``False`` + + Returns + ------- + numpy.ndarray + 2x2 response matrix + """ + branch = {"p1": self.p1, "m1": self.m1, "p2": self.p2, "m2": self.m2} + h2 = 2 * self._step + return np.array( + [ + [ + ( + self._mean_ns(comp, sel[plus], branch[plus] if joint else None) + - self._mean_ns( + comp, sel[minus], branch[minus] if joint else None + ) + ) + / h2 + for plus, minus in (("p1", "m1"), ("p2", "m2")) + ] + for comp in ("g1", "g2") + ] + ) + + def _joint_response(self): + """Shear and selection response in one go, see + ``_response_from_selections``. Alternative to R.""" + self.R_joint = self._response_from_selections(self.mask_dict, joint=True) + + def _selection_response_std(self, n_realization=100, remove_size=0.1, seed=42): + """Resampling error on ``R_selection``. + + Why this needs its own estimator, rather than the one the shear term + uses: R_selection is a DIFFERENCE OF TWO MEANS over different subsets + of the same objects, and those subsets overlap almost entirely -- they + differ only in objects near the selection boundary. Resampling the two + independently would treat them as unrelated samples and inflate the + error by orders of magnitude. So OBJECTS are resampled once per + realization and both masked means are recomputed from the same draw, + which preserves their correlation, and the four components share a draw + so their errors are mutually consistent too. + + Why it matters at all: R_selection = Delta / 2h with h = 0.01, so it + divides a tiny difference of means by 0.02 -- a 50x amplification of + whatever noise is in . An R_selection of 0.02 corresponds to a + difference of only 4e-4 in mean ellipticity. Its scatter therefore has + to be measured before any m-bias difference is attributed to it. + + Seeded by default: this is a diagnostic that gets compared between + runs, so it must not change when nothing else did. + """ + rng = np.random.default_rng(seed) + n_obj = len(self.ns["g1"]) + + # mask_dict holds INTEGER INDEX arrays (_masking_gal stores + # np.where(...)[0]), which index the full catalogue directly. Resampling + # needs the opposite question -- "is object i selected?" -- so convert + # once to boolean membership over all n_obj. Accepts a boolean mask too, + # in case a masking variant ever stores one. + def as_membership(m): + m = np.asarray(m) + if m.dtype == bool and m.shape[0] == n_obj: + return m + out = np.zeros(n_obj, dtype=bool) + out[m] = True + return out + + masks = {k: as_membership(self.mask_dict[k]) for k in ("p1", "m1", "p2", "m2")} + keep = int(n_obj * (1 - remove_size)) + if keep < 2: + self.R_selection_std = np.full((2, 2), np.nan) + return + + weighted = not ( + self._global_R_weight is None or self._global_R_weight == "None" + ) + h2 = 2 * self._step + + def mean_at(comp, mask, idx): + sel = idx[mask[idx]] + if len(sel) == 0: + return np.nan + values = self.ns[comp][sel] + if weighted: + return np.average(values, weights=self.ns[self._global_R_weight][sel]) + return np.mean(values) + + est = [] + for _ in range(n_realization): + idx = rng.choice(n_obj, keep, replace=True) + est.append([ + (mean_at("g1", masks["p1"], idx) - mean_at("g1", masks["m1"], idx)) / h2, + (mean_at("g1", masks["p2"], idx) - mean_at("g1", masks["m2"], idx)) / h2, + (mean_at("g2", masks["p1"], idx) - mean_at("g2", masks["m1"], idx)) / h2, + (mean_at("g2", masks["p2"], idx) - mean_at("g2", masks["m2"], idx)) / h2, + ]) + s = np.nanstd(np.array(est), axis=0) + self.R_selection_std = np.array([[s[0], s[1]], [s[2], s[3]]]) + + def response_jackknife(self, groups): + """Full responses R and R_joint with one group left out. + + Recomputes R = R_shear + R_selection and R_joint, with the same + estimators and weighting as the full-sample ones, once per group + label, dropping that group's objects from every metacal branch. Used + for a leave-one-tile-out jackknife: the m-bias step recalibrates each + jackknife sample with its own response, so the noise in the response + reaches the error on m. + + Parameters + ---------- + groups : array-like + group label (e.g. tile ID) per object of the metacal input, that + is, aligned with ``data[mask]`` as passed to the constructor + + Returns + ------- + labels : numpy.ndarray + sorted unique group labels + R_jk : numpy.ndarray + full response with that group left out, shape (n_labels, 2, 2) + R_joint_jk : numpy.ndarray + joint response with that group left out, shape (n_labels, 2, 2) + """ + groups = np.asarray(groups) + if len(groups) != len(self.ns["g1"]): + raise ValueError( + f"groups has {len(groups)} entries, metacal input has" + + f" {len(self.ns['g1'])}" + ) + labels = np.unique(groups) + + if self._global_R_weight is None or self._global_R_weight == "None": + weight = np.ones(len(groups)) + else: + weight = self.ns[self._global_R_weight] + + # Shear response: per-object values on the noshear selection + ma = self.mask_dict["ns"] + R_obj = self.R_shear + grp_ns = groups[ma] + w_ns = weight[ma] + + R_jk = np.empty((len(labels), 2, 2)) + R_joint_jk = np.empty((len(labels), 2, 2)) + for idx, label in enumerate(labels): + keep = grp_ns != label + R_shear = np.average(R_obj[:, :, keep], axis=2, weights=w_ns[keep]) + + # Sheared selections without this group + sel = { + key: self.mask_dict[key][groups[self.mask_dict[key]] != label] + for key in ("p1", "m1", "p2", "m2") + } + R_jk[idx] = R_shear + self._response_from_selections(sel) + R_joint_jk[idx] = self._response_from_selections(sel, joint=True) + + return labels, R_jk, R_joint_jk + def _total_response(self): """Add docstring. diff --git a/src/sp_validation/image_sims.py b/src/sp_validation/image_sims.py index b29ab368..233c58ad 100644 --- a/src/sp_validation/image_sims.py +++ b/src/sp_validation/image_sims.py @@ -49,6 +49,13 @@ def _load_cat(path, w_cols): sim calibration); any other entry is read as a FITS column name. The weights come back as a dict keyed by scheme so one catalogue load serves every scheme in a multi-weight run. + + If the header has the joint response ``RJ_ij`` (written by the + calibration stage), the shear recalibrated with it is loaded as + ``e1_joint``/``e2_joint``. If the catalogue has a ``TILE_ID`` column and an + ``R_JK`` HDU (the leave-one-tile-out responses), the inputs for a + jackknife over tiles are loaded under ``cat["jk"]``; see + ``_load_jackknife``. """ with fits.open(path) as hdul: data = hdul[1].data @@ -65,9 +72,81 @@ def _load_cat(path, w_cols): if _is_unweighted(scheme) else data[scheme].copy() ) + has_joint = "RJ_11" in hdul[0].header + has_jk = "TILE_ID" in data.columns.names and "R_JK" in hdul + if has_joint or has_jk: + R, g_off = _uncalibrated(hdul, cat, path) + if has_joint: + R_joint = _matrix(hdul[0].header, "RJ") + cat["e1_joint"], cat["e2_joint"] = np.linalg.inv(R_joint) @ g_off + if has_jk: + cat["jk"] = _load_jackknife(hdul, R, R_joint if has_joint else None, g_off) return cat +def _matrix(row, key): + """2x2 matrix from the entries ``_11`` ... ``_22`` of ``row``.""" + return np.array( + [[row[f"{key}_11"], row[f"{key}_12"]], [row[f"{key}_21"], row[f"{key}_22"]]] + ) + + +def _uncalibrated(hdul, cat, path): + """Response and uncalibrated shear, to recalibrate a catalogue. + + The calibrated shear is ``e = R^-1 g_uncal - s``, where the shift ``s`` is + the additive correction (zero when ``additive_correction`` is off, as for + constant-shear sims). Returns R and ``g_off = g_uncal - R s``, so that a + response R' recalibrates as ``e' = R'^-1 g_off``, which gives back ``e`` + for R' = R. + """ + data = hdul[1].data + R = _matrix(hdul[0].header, "R") + g_uncal = np.array([data["e1_uncal"], data["e2_uncal"]], dtype=float) + + shift = np.linalg.inv(R) @ g_uncal - np.array([cat["e1"], cat["e2"]]) + if np.any(np.ptp(shift, axis=1) > 1e-8): + raise ValueError( + f"{path}: e1/e2 are not R^-1 e_uncal minus a constant, cannot" + + " recalibrate" + ) + return R, g_uncal - (R @ shift.mean(axis=1))[:, None] + + +def _load_jackknife(hdul, R, R_joint, g_off): + """Inputs to recalibrate a catalogue with a leave-one-tile-out response. + + A jackknife sample without tile t is recalibrated as + ``e_t = R_t^-1 g_off``, see ``_uncalibrated``; the same for the joint + response, if available (``R_joint`` not ``None`` and the ``RJ_ij`` + columns present). + """ + data = hdul[1].data + jk = hdul["R_JK"].data + tiles = [str(t).strip() for t in jk["TILE_ID"]] + + resp = {"default": (R, "R")} + if R_joint is not None and "RJ_11" in jk.columns.names: + resp["joint"] = (R_joint, "RJ") + return { + "tile": np.char.strip(np.asarray(data["TILE_ID"]).astype(str)), + "g_off": g_off, + "resp": { + method: { + "R": R_full, + "R_jk": {t: _matrix(jk[i], key) for i, t in enumerate(tiles)}, + } + for method, (R_full, key) in resp.items() + }, + } + + +def _jackknife_std(x): + """Leave-one-out jackknife standard deviation.""" + n = len(x) + return np.sqrt((n - 1) / n * np.sum((x - np.mean(x)) ** 2)) + + class ImageSimMBias: """Compute multiplicative and additive shear bias from image simulations. @@ -179,7 +258,7 @@ def print_mean_ellipticities(self): mean_e2 = np.average(cat["e2"], weights=cat["w"][scheme]) print(f" {name}: = {mean_e1:+.5f} = {mean_e2:+.5f}") - def _m_c_pair(self, name_p, name_m, comp, verbose=True): + def _m_c_pair(self, name_p, name_m, comp, verbose=True, method="default"): """Compute m and c for one shear pair and component (0=g1, 1=g2). Paired ("pool") estimator. The +g and -g simulations inject opposite @@ -201,8 +280,12 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): weighted means. The paired bootstrap likewise cannot be applied (the two arrays generally have different lengths), so each side is resampled independently per replicate. + + ``method`` selects the response the shear is calibrated with: + "default" (R = R_shear + R_selection, the catalogue's e1/e2) or + "joint" (R_joint, see ``metacal._response_from_selections``). """ - e_key = f"e{comp + 1}" + e_key = f"e{comp + 1}" + ("_joint" if method == "joint" else "") if self.pair_match: # Match the +g and -g sims to each other: same galaxies, opposite @@ -287,10 +370,76 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): m_boot[i] = (ep_b - em_b) / (2 * self.g_in) - 1 c_boot[i] = (ep_b + em_b) / 2 - res[scheme] = (m, np.std(m_boot), c, np.std(c_boot)) + res[scheme] = { + "m": m, + "m_err": np.std(m_boot), + "c": c, + "c_err": np.std(c_boot), + } + + if all( + method in self.cats[name].get("jk", {}).get("resp", {}) + for name in (name_p, name_m) + ): + self._jackknife_pair(name_p, name_m, comp, idx_p, idx_m, res, method) + elif verbose: + print(" no tile IDs or R_JK on input, skipping tile jackknife") return res + def _jackknife_pair(self, name_p, name_m, comp, idx_p, idx_m, res, method): + """Leave-one-tile-out jackknife errors on m and c, added to ``res``. + + Each sample drops one tile from both the +g and the -g catalogue, and + recalibrates each catalogue with the response computed without that + tile. The error thus includes the noise of R_shear and R_selection, + which the bootstrap over calibrated ellipticities (fixed R) misses. + Tiles are resampled as a whole, which keeps correlations within a tile. + ``method`` selects the response, as in ``_m_c_pair``. + """ + jk_p = self.cats[name_p]["jk"] + jk_m = self.cats[name_m]["jk"] + resp_p = jk_p["resp"][method] + resp_m = jk_m["resp"][method] + tile_p = jk_p["tile"][idx_p] + tile_m = jk_m["tile"][idx_m] + g_off_p = jk_p["g_off"][:, idx_p] + g_off_m = jk_m["g_off"][:, idx_m] + tiles = np.union1d(tile_p, tile_m) + + m_jk = {scheme: np.empty(len(tiles)) for scheme in self.w_cols} + c_jk = {scheme: np.empty(len(tiles)) for scheme in self.w_cols} + for i, tile in enumerate(tiles): + # Recalibrated component comp, with the tile's leave-out response + # (the full R if the tile is absent from that catalogue) + row_p = np.linalg.inv(resp_p["R_jk"].get(tile, resp_p["R"]))[comp] + row_m = np.linalg.inv(resp_m["R_jk"].get(tile, resp_m["R"]))[comp] + e_p = row_p @ g_off_p + e_m = row_m @ g_off_m + + for scheme in self.w_cols: + w_p = self.cats[name_p]["w"][scheme][idx_p] + w_m = self.cats[name_m]["w"][scheme][idx_m] + if self.pair_match: + keep = (tile_p != tile) & (tile_m != tile) + w = 0.5 * (w_p + w_m)[keep] + d = (e_p[keep] - e_m[keep]) / (2 * self.g_in) - 1 + s = (e_p[keep] + e_m[keep]) / 2 + m_jk[scheme][i] = np.average(d, weights=w) + c_jk[scheme][i] = np.average(s, weights=w) + else: + keep_p = tile_p != tile + keep_m = tile_m != tile + mean_ep = np.average(e_p[keep_p], weights=w_p[keep_p]) + mean_em = np.average(e_m[keep_m], weights=w_m[keep_m]) + m_jk[scheme][i] = (mean_ep - mean_em) / (2 * self.g_in) - 1 + c_jk[scheme][i] = (mean_ep + mean_em) / 2 + + for scheme in self.w_cols: + res[scheme]["m_err_jk"] = _jackknife_std(m_jk[scheme]) + res[scheme]["c_err_jk"] = _jackknife_std(c_jk[scheme]) + res[scheme]["n_jk"] = len(tiles) + def run(self, verbose=True): """Compute m and c for both shear components and every weight scheme. @@ -301,28 +450,53 @@ def run(self, verbose=True): m2, m2_err, c2, c2_err`` for each weight scheme. The primary (first) scheme's keys are also mirrored at the top level, so a reader that wants the headline m/c never has to know the scheme - name. + name. If the inputs support a tile jackknife, the keys + ``m1_err_jk, c1_err_jk, n1_jk`` (and for comp 2) are added. If the + inputs carry the joint response, ``results["joint"]["weights"]`` + holds the same keys with the shear calibrated by R_joint. """ - results = {"weights": {scheme: {} for scheme in self.w_cols}} - for name_p, name_m, comp in self.pairs: - label = f"g{comp + 1}" - if verbose: - print(f"\n--- {label}: {name_p} / {name_m} ---") - res = self._m_c_pair(name_p, name_m, comp, verbose=verbose) - for scheme, (m, m_err, c, c_err) in res.items(): - w = results["weights"][scheme] - w[f"m{comp + 1}"] = m - w[f"m{comp + 1}_err"] = m_err - w[f"c{comp + 1}"] = c - w[f"c{comp + 1}_err"] = c_err - if verbose: - print( - f" [{scheme}] m{comp + 1} = {m:.4f} ± {m_err:.4f}" - f" c{comp + 1} = {c:.4f} ± {c_err:.4f}" - ) + results = {"weights": self._run_method("default", verbose)} + + names = {name for pair in self.pairs for name in pair[:2]} + if all("e1_joint" in self.cats[name] for name in names): + results["joint"] = {"weights": self._run_method("joint", verbose)} + elif verbose: + print("\nNo joint response on input, skipping method 'joint'") # Mirror the primary (first) scheme's m/c at the top level: the headline # result reads out without knowing the scheme name, and a downstream # gate keyed on the old flat keys still finds them. results.update(results["weights"][self.w_cols[0]]) return results + + def _run_method(self, method, verbose): + """m and c per weight scheme, for the response ``method``.""" + weights = {scheme: {} for scheme in self.w_cols} + for name_p, name_m, comp in self.pairs: + label = f"g{comp + 1}" + if verbose: + print(f"\n--- {label}: {name_p} / {name_m}, response {method} ---") + res = self._m_c_pair(name_p, name_m, comp, verbose=verbose, method=method) + for scheme, r in res.items(): + w = weights[scheme] + k = comp + 1 + w[f"m{k}"] = r["m"] + w[f"m{k}_err"] = r["m_err"] + w[f"c{k}"] = r["c"] + w[f"c{k}_err"] = r["c_err"] + if "n_jk" in r: + w[f"m{k}_err_jk"] = r["m_err_jk"] + w[f"c{k}_err_jk"] = r["c_err_jk"] + w[f"n{k}_jk"] = r["n_jk"] + if verbose: + print( + f" [{scheme}] m{k} = {r['m']:.4f} ± {r['m_err']:.4f}" + f" c{k} = {r['c']:.4f} ± {r['c_err']:.4f}" + ) + if "n_jk" in r: + print( + f" [{scheme}] tile jackknife ({r['n_jk']} tiles):" + f" m{k} ± {r['m_err_jk']:.4f}" + f" c{k} ± {r['c_err_jk']:.4f}" + ) + return weights diff --git a/src/sp_validation/tests/test_calibration.py b/src/sp_validation/tests/test_calibration.py index 0a66aed5..982be02f 100644 --- a/src/sp_validation/tests/test_calibration.py +++ b/src/sp_validation/tests/test_calibration.py @@ -413,3 +413,84 @@ def test_mask_gal_SNR_boolean_mask(): mask_shifted = mask_gal_SNR(SNR, snr_min=12.0, snr_max=500.0) assert mask[2] and not mask_shifted[2] assert not np.array_equal(mask, mask_shifted) + + +def _build_noisy_ngmix_catalog(n=600, n_groups=5, step=0.01, seed=1): + """NGMIX catalogue with scatter, so objects cross the cuts between + branches and the selection response is non-zero; plus a group label.""" + rng = np.random.default_rng(seed) + g1 = rng.normal(0, 0.3, n) + g2 = rng.normal(0, 0.3, n) + shifts = { + "NOSHEAR": (0.0, 0.0), + "1P": (step, 0.0), + "1M": (-step, 0.0), + "2P": (0.0, step), + "2M": (0.0, -step), + } + cols = {} + for var in _VARIANTS: + dg1, dg2 = shifts[var] + cols[f"NGMIX_G1_{var}"] = g1 + 0.8 * dg1 + 0.05 * dg2 + cols[f"NGMIX_G2_{var}"] = g2 + 0.8 * dg2 - 0.03 * dg1 + cols[f"NGMIX_FLAGS_{var}"] = np.zeros(n, dtype=int) + # SNR near the cut, shifting with the branch -> selection response + cols[f"NGMIX_FLUX_{var}"] = rng.uniform(5, 30, n) * (1 + 3 * (dg1 + dg2)) + cols[f"NGMIX_FLUX_ERR_{var}"] = np.ones(n) + cols[f"NGMIX_T_{var}"] = rng.uniform(0.3, 3.5, n) + cols[f"NGMIX_T_ERR_{var}"] = np.full(n, 0.1) + cols[f"NGMIX_T_PSF_RECONV_{var}"] = np.ones(n) + cols["NGMIX_G1_ERR_NOSHEAR"] = rng.uniform(0.01, 0.1, n) + cols["NGMIX_G2_ERR_NOSHEAR"] = rng.uniform(0.01, 0.1, n) + groups = np.array([f"t{i}" for i in rng.integers(0, n_groups, n)]) + return Table(cols), groups + + +@pytest.mark.parametrize("weight", [None, "w"]) +def test_metacal_response_jackknife_matches_leave_out_rebuild(weight): + """Each leave-one-group-out R equals R of a metacal built without that + group, for both the shear and the selection term.""" + data, groups = _build_noisy_ngmix_catalog() + kw = dict(size_corr_ell=False, global_R_weight=weight, step=0.01) + mcal = metacal(data, np.ones(len(groups), dtype=bool), **kw) + assert np.all(np.abs(mcal.R_selection) > 0) + + labels, R_jk, R_joint_jk = mcal.response_jackknife(groups) + npt.assert_array_equal(labels, np.unique(groups)) + for label, R, R_joint in zip(labels, R_jk, R_joint_jk): + ref = metacal(data, groups != label, **kw) + npt.assert_allclose(R, ref.R, rtol=1e-12, atol=1e-14) + npt.assert_allclose(R_joint, ref.R_joint, rtol=1e-12, atol=1e-14) + assert not np.allclose(R, mcal.R) + assert not np.allclose(R_joint, mcal.R_joint) + + +@pytest.mark.parametrize("weight", [None, "w"]) +def test_metacal_joint_response(weight): + """R_joint is the difference of sheared-branch means, each over its own + sheared selection; it differs from R = R_shear + R_selection only at + second order.""" + data, _ = _build_noisy_ngmix_catalog() + mcal = metacal( + data, + np.ones(len(data), dtype=bool), + size_corr_ell=False, + global_R_weight=weight, + step=0.01, + ) + w = mcal.ns["w"] if weight else np.ones(len(data)) + ma = mcal.mask_dict + + def mean(branch, comp, key): + return np.average(branch[comp][ma[key]], weights=w[ma[key]]) + + R11 = (mean(mcal.p1, "g1", "p1") - mean(mcal.m1, "g1", "m1")) / 0.02 + R12 = (mean(mcal.p2, "g1", "p2") - mean(mcal.m2, "g1", "m2")) / 0.02 + R21 = (mean(mcal.p1, "g2", "p1") - mean(mcal.m1, "g2", "m1")) / 0.02 + R22 = (mean(mcal.p2, "g2", "p2") - mean(mcal.m2, "g2", "m2")) / 0.02 + npt.assert_allclose(mcal.R_joint, [[R11, R12], [R21, R22]], rtol=1e-12) + + # In this catalogue every object has the same shear response, so + # over sel_p = over sel_p + const and the two estimators + # agree to rounding + npt.assert_allclose(mcal.R_joint, mcal.R, rtol=1e-10, atol=1e-12) diff --git a/src/sp_validation/tests/test_image_sims.py b/src/sp_validation/tests/test_image_sims.py index ef3e3b14..65011938 100644 --- a/src/sp_validation/tests/test_image_sims.py +++ b/src/sp_validation/tests/test_image_sims.py @@ -251,3 +251,222 @@ def measured(g1_in, g2_in): assert abs(res[f"m{comp}"] - M_TRUE) < 5 * res[f"m{comp}_err"] # ...and its error is far below what an unpaired estimator would give. assert res[f"m{comp}_err"] < 0.1 * shape_noise_floor + + +def _write_jk_cat(path, ra, dec, g_uncal, tiles, R, R_jk, RJ=None, RJ_jk=None): + """Calibrated catalogue with TILE_ID and a leave-one-tile-out R_JK HDU; + optionally with the joint response (header RJ_ij, columns RJ_ij).""" + e = np.linalg.inv(R) @ g_uncal + cols = [ + fits.Column(name=name, array=arr, format="D") + for name, arr in ( + ("RA", ra), + ("Dec", dec), + ("e1", e[0]), + ("e2", e[1]), + ("e1_uncal", g_uncal[0]), + ("e2_uncal", g_uncal[1]), + ) + ] + cols.append(fits.Column(name="TILE_ID", array=tiles, format="7A")) + primary = fits.PrimaryHDU() + for i in (1, 2): + for j in (1, 2): + primary.header[f"R_{i}{j}"] = R[i - 1, j - 1] + if RJ is not None: + primary.header[f"RJ_{i}{j}"] = RJ[i - 1, j - 1] + labels = sorted(R_jk) + jk_cols = [fits.Column(name="TILE_ID", array=labels, format="7A")] + for key, resp in (("R", R_jk), ("RJ", RJ_jk)): + if resp is None: + continue + for i in (1, 2): + for j in (1, 2): + jk_cols.append( + fits.Column( + name=f"{key}_{i}{j}", + array=[resp[t][i - 1, j - 1] for t in labels], + format="D", + ) + ) + fits.HDUList( + [ + primary, + fits.BinTableHDU.from_columns(cols), + fits.BinTableHDU.from_columns(jk_cols, name="R_JK"), + ] + ).writeto(path, overwrite=True) + + +def test_mbias_tile_jackknife_recalibrates_per_tile(tmp_path): + """The tile jackknife drops one tile from both catalogues, recalibrates + each with its leave-out response, and uses the (N-1)/N jackknife factor; + checked against a brute-force computation.""" + num = 3 + n_tiles = 6 + rng = np.random.default_rng(5) + ra = 30.0 + rng.uniform(0, 0.1, N_GAL) + dec = rng.uniform(0, 0.1, N_GAL) + tiles = np.array([f"{100 + i}.200" for i in rng.integers(0, n_tiles, N_GAL)]) + labels = np.unique(tiles) + e_int = rng.normal(0, 0.3, (2, N_GAL)) + + R_true = np.array([[0.75, 0.01], [-0.02, 0.72]]) + g_in = { + "1z2z": (0, 0), + "1p2z": (A, 0), + "1m2z": (-A, 0), + "1z2p": (0, A), + "1z2m": (0, -A), + } + truth = {} + for idx, (name, g) in enumerate(g_in.items()): + g_uncal = R_true @ (e_int + np.array(g)[:, None] * (1 + M_TRUE)) + g_uncal += rng.normal(0, 0.01, (2, N_GAL)) + R = R_true + rng.normal(0, 0.005, (2, 2)) + R_jk = {t: R + rng.normal(0, 0.01, (2, 2)) for t in labels} + sim_dir = tmp_path / f"{name}_grid_{num}" + sim_dir.mkdir() + _write_jk_cat(sim_dir / "cat.fits", ra, dec, g_uncal, tiles, R, R_jk) + truth[name] = (g_uncal, R_jk) + + config = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "w_cols": ["none"], + "n_bootstrap": 20, + "pair_match": True, + "bootstrap_seed": 42, + } + mb = ImageSimMBias(config) + mb.load_catalogs(verbose=False) + res = mb.run(verbose=False) + + for name_p, name_m, comp in (("1p2z", "1m2z", 0), ("1z2p", "1z2m", 1)): + m_jk = [] + for t in labels: + keep = tiles != t + (gp, Rp), (gm, Rm) = truth[name_p], truth[name_m] + ep = np.linalg.solve(Rp[t], gp)[comp][keep] + em = np.linalg.solve(Rm[t], gm)[comp][keep] + m_jk.append(np.mean((ep - em) / (2 * A)) - 1) + m_jk = np.array(m_jk) + err = np.sqrt((n_tiles - 1) / n_tiles * np.sum((m_jk - m_jk.mean()) ** 2)) + k = comp + 1 + assert res[f"n{k}_jk"] == n_tiles + npt.assert_allclose(res[f"m{k}_err_jk"], err, rtol=1e-10) + # TEETH: recalibrating per tile matters; a fixed-R tile jackknife + # gives a different error. + (gp, _), (gm, _) = truth[name_p], truth[name_m] + ep_fix = np.linalg.solve(mb.cats[name_p]["jk"]["resp"]["default"]["R"], gp)[comp] + em_fix = np.linalg.solve(mb.cats[name_m]["jk"]["resp"]["default"]["R"], gm)[comp] + m_fix = np.array( + [np.mean((ep_fix - em_fix)[tiles != t] / (2 * A)) - 1 for t in labels] + ) + err_fix = np.sqrt((n_tiles - 1) / n_tiles * np.sum((m_fix - m_fix.mean()) ** 2)) + assert not np.isclose(res[f"m{k}_err_jk"], err_fix, rtol=1e-3) + + +def test_mbias_no_tile_jackknife_without_inputs(tmp_path): + """Catalogues without TILE_ID / R_JK give bootstrap errors only.""" + num = 8 + _make_grid(tmp_path, num) + config = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "w_cols": ["w_des"], + "n_bootstrap": 10, + "pair_match": True, + "bootstrap_seed": 42, + } + mb = ImageSimMBias(config) + mb.load_catalogs(verbose=False) + res = mb.run(verbose=False) + assert not any(key.endswith("_jk") for key in res) + + +def test_mbias_joint_response(tmp_path): + """With the joint response on input, m/c are computed a second time with + the shear recalibrated by R_joint, including the tile jackknife; the + default results are unchanged.""" + num = 4 + n_tiles = 5 + rng = np.random.default_rng(9) + ra = 30.0 + rng.uniform(0, 0.1, N_GAL) + dec = rng.uniform(0, 0.1, N_GAL) + tiles = np.array([f"{100 + i}.200" for i in rng.integers(0, n_tiles, N_GAL)]) + labels = np.unique(tiles) + e_int = rng.normal(0, 0.3, (2, N_GAL)) + + R_true = np.array([[0.75, 0.01], [-0.02, 0.72]]) + g_in = { + "1z2z": (0, 0), + "1p2z": (A, 0), + "1m2z": (-A, 0), + "1z2p": (0, A), + "1z2m": (0, -A), + } + truth = {} + for name, g in g_in.items(): + g_uncal = R_true @ (e_int + np.array(g)[:, None] * (1 + M_TRUE)) + R = R_true + rng.normal(0, 0.005, (2, 2)) + RJ = R + rng.normal(0, 0.005, (2, 2)) + R_jk = {t: R + rng.normal(0, 0.01, (2, 2)) for t in labels} + RJ_jk = {t: RJ + rng.normal(0, 0.01, (2, 2)) for t in labels} + sim_dir = tmp_path / f"{name}_grid_{num}" + sim_dir.mkdir() + _write_jk_cat( + sim_dir / "cat.fits", ra, dec, g_uncal, tiles, R, R_jk, RJ, RJ_jk + ) + truth[name] = (g_uncal, R, RJ, RJ_jk) + + config = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "w_cols": ["none"], + "n_bootstrap": 20, + "pair_match": True, + "bootstrap_seed": 42, + } + mb = ImageSimMBias(config) + mb.load_catalogs(verbose=False) + res = mb.run(verbose=False) + joint = res["joint"]["weights"]["none"] + + for name_p, name_m, comp in (("1p2z", "1m2z", 0), ("1z2p", "1z2m", 1)): + (gp, Rp, RJp, RJp_jk), (gm, Rm, RJm, RJm_jk) = truth[name_p], truth[name_m] + k = comp + 1 + + # Central values: default with R, joint with R_joint + for resp_p, resp_m, out in ((Rp, Rm, res), (RJp, RJm, joint)): + ep = np.linalg.solve(resp_p, gp)[comp] + em = np.linalg.solve(resp_m, gm)[comp] + npt.assert_allclose(out[f"m{k}"], np.mean((ep - em) / (2 * A)) - 1) + npt.assert_allclose(out[f"c{k}"], np.mean((ep + em) / 2), atol=1e-15) + + # Joint jackknife uses the joint leave-out responses + m_jk = np.array( + [ + np.mean( + ( + np.linalg.solve(RJp_jk[t], gp)[comp] + - np.linalg.solve(RJm_jk[t], gm)[comp] + )[tiles != t] + / (2 * A) + ) + - 1 + for t in labels + ] + ) + err = np.sqrt((n_tiles - 1) / n_tiles * np.sum((m_jk - m_jk.mean()) ** 2)) + npt.assert_allclose(joint[f"m{k}_err_jk"], err, rtol=1e-10) + assert joint[f"m{k}"] != res[f"m{k}"] diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 9133a7e6..b93f423a 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -1,49 +1,29 @@ -# Image-simulation m-bias workflow configuration. -# -# OPERATIONAL keys default here and nowhere else -- the .smk reads them bare, so -# this file is their single home. SCIENCE keys have no default anywhere and -# appear below only as commented template lines: they fix the estimator's -# behaviour, so each run must state them in a run config layered on top. A run -# config that omits one fails at DAG parse, naming the key; an unknown key under -# `image_sims:` fails as a typo. +# Config file for sp_validation post-processing of image simulations. +# Inputs: final catalogues for 4 sheared versions. +# Output: Multiplicative bias estimation. image_sims: - # --- container -------------------------------------------------------- - # One image for the whole chain: the sp_validation image is built FROM the - # ShapePipe image, so it carries both stacks. CI builds it from the uv lock -- - # an unlocked build drifts NumPy past numba's ceiling and the ngmix stage dies - # ("Numba needs NumPy 2.4 or less"). - # - # `null` means the workflow's one image: your own .sif if you have pulled one - # with `spv-container pull`, else the registry tag for Snakemake to autopull - # (workflow/README.md). Neither is repeated here. - # Override with a local .sif path, or with a branch tag - # (docker://ghcr.io/cosmostat/sp_validation:) to run a - # branch's own CI image. - sif: null - - # --- repositories ----------------------------------------------------- + # --- repositories (REQUIRED in the run config; no default) ------------- # Bound into the image; both repos' src go on PYTHONPATH so this branch's - # code wins over the baked copies. - shapepipe_repo: /n17data/cdaley/unions/code/shapepipe - sp_validation_repo: /n17data/cdaley/unions/code/sp_validation + # code wins over the baked copies. Per-user checkouts, so there is nothing + # sensible to default to -- a default here would silently run someone else's + # code. A run config that omits one fails at parse, naming it. + # + # shapepipe_repo: /path/to/shapepipe + # sp_validation_repo: /path/to/sp_validation # --- data and run directories ---------------------------------------- - # grids_base is the run/output root: one sub-directory per simulation. - grids_base: /n17data/cdaley/unions/scratch-wf/imsims-run/grids + # grids_base is the run/output root (one sub-directory per simulation), and + # is REQUIRED in the run config -- it is where this user's campaign writes. + # + # grids_base: /path/to/grids input_sims_base: /n09data/hervas/skills_out psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle # --- simulation grid -------------------------------------------------- sims_type: grid # 'grid' -> *_grid_{num}; anything else -> *_{num} [operational] num: 1 - # Optional: pull exposures from a different realization than tiles (tiles - # from `{branch}_{num}`, exposures from `{branch}_{exp_num}`; unset -> - # exposures follow `num`). Non-grid sims only. Fabian's blended production - # split: tiles from _1, exposures from _5 (the clean all-real exposure set; - # _1.._4 carry signal-free grid-sims SP_exp cross-links). - # exp_num: 5 # Branches this run requests: the unsheared reference plus the four +/- # sheared branches. Injected shear is NOT set here -- it is parsed from each # branch's basic_info.txt by im_manifest. [operational] @@ -86,11 +66,13 @@ image_sims: # every later tile group on that node. Set a shared directory for the star # sims (all branches and runs can share one root). [operational] tile_store_root: null - # Extract/calibrate scripts run from the sp_validation repo checkout (branch - # code, not the baked copies). Point elsewhere for a different checkout. - # [operational] - extract_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/extract_info.py - calibrate_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/calibrate_comprehensive_cat.py + # Extract/calibrate scripts default to the sp_validation_repo checkout above + # (as compute_m_bias_image_sims.py already did), so branch code runs rather + # than the copies baked into the image. Set one only to point at a DIFFERENT + # checkout. + # + # extract_script: /other/checkout/scripts/calibration/extract_info.py + # calibrate_script: /other/checkout/scripts/calibration/calibrate_comprehensive_cat.py # --- science knobs (REQUIRED in the run config; no default) ----------- # Copy these into your run config and set them. There is deliberately no diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 229d8381..5415caf5 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -57,7 +57,7 @@ default-resources: slurm_partition: "comp,pscomp" runtime: "60m" cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n36'" + slurm_extra: "'--exclude=n17,n35,n36'" # Snakemake refuses a real run on a remote executor without a job bound. jobs: 100 diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index e576c39b..4d088470 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -54,16 +54,24 @@ _OPERATIONAL_KEYS = { "sp_jobs", "clean_exposures", "tile_store_root", - "extract_script", - "calibrate_script", } # Optional keys: recognized but not required; defaults applied in code below. _OPTIONAL_KEYS = { - "exp_num", + # Overrides only: each has a working default derived below from + # sp_validation_repo, so pointing at a different checkout is opt-in and a + # missing key cannot silently select someone else's code. + "extract_script", + "calibrate_script", + # The container image. Optional because it has a working default chain + # (sp_validation.container.resolve_image: local sandbox, then local sif, + # then the registry tag), unlike the structural keys below, which have no + # sensible default at all. On candide it comes from the profile, whose + # `config:` sets the cluster's image -- a machine-specific path belongs + # with the machine, not in this shared file. + "sif", } # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { - "sif", "shapepipe_repo", "sp_validation_repo", "grids_base", @@ -107,7 +115,7 @@ if _missing_structural: # workflow/Snakefile, whose module default is the cosmology image (no ShapePipe # stack). Binds come from the driving profile's ``apptainer-args``. A null # ``sif`` resolves to the workflow's one image (see workflow/image_sims/config.yaml). -SIF = common.resolve_container(IMSIM["sif"]) +SIF = common.resolve_container(IMSIM.get("sif")) # --- repositories (bound into the image; branch code overrides) ----------- SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] @@ -167,13 +175,17 @@ CLEAN_EXPOSURES = bool(IMSIM["clean_exposures"]) TILE_STORE_ROOT = IMSIM["tile_store_root"] # ShapePipe scripts live in the ShapePipe repo (also baked into its image). -CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" # Extract/calibrate run from the sp_validation *repo* checkout (bind-mounted), # not the baked copies: the container tracks the branch but lags it, and the # image-sims path needs branch-only fixes (star-catalogue-optional extract, # FITS-aware CalibrateCat.read_cat). Overridable for a different checkout. -EXTRACT_INFO = IMSIM["extract_script"] -CALIBRATE = IMSIM["calibrate_script"] +EXTRACT_INFO = IMSIM.get( + "extract_script", f"{SPV_REPO}/scripts/calibration/extract_info.py" +) +CALIBRATE = IMSIM.get( + "calibrate_script", + f"{SPV_REPO}/scripts/calibration/calibrate_comprehensive_cat.py", +) # m-bias is *this branch's* extracted core, injected on PYTHONPATH. COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" @@ -184,8 +196,8 @@ COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" # * PYTHONPATH prepends both repos' ``src`` so Python resolves the worktree # build ahead of the copies baked into the image -- the branch's code runs # without an image rebuild. Packages only: the bash and python entry points -# are invoked at the repo paths from config (RUN_JOB, CREATE_FINAL_CAT, -# EXTRACT_INFO, ...), not shadowed by PYTHONPATH. +# are invoked at the repo paths from config (EXTRACT_INFO, +# CALIBRATE, ...), not shadowed by PYTHONPATH. # * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded # via getexpanded) at this run's PSF dictionary. # * OMP_NUM_THREADS=1 rides here rather than in the SLURM profile: the chain @@ -237,7 +249,7 @@ def sp_run_config(sim): localrules: - im_shapepipe, + im_pipeline, wildcard_constraints: @@ -263,11 +275,6 @@ rule im_shapepipe_all: expand(f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", sim=SIMS), -rule im_merge_all: - input: - expand(f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", sim=SIMS), - - rule im_extract_all: input: expand( @@ -381,29 +388,25 @@ rule im_pipeline: "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " "-p {params.psf} -N {params.n_smp}" +def merged_cat(wc): + """This sim's merged catalogue. -rule im_merge: - """Merge per-tile ShapePipe catalogues into final_cat_{sim}.hdf5. + ShapePipe's own workflow (``merge_final_cats``, shapepipe #891) publishes it + under ``product/`` with the rest of the campaign's products, and that is the + only copy surviving ``clean_tile``. This workflow no longer merges -- the + ``im_merge`` rule that wrote one at the branch root was dropped once the + ShapePipe side gained the step. - ``create_final_cat.py`` lives in the ShapePipe repo/image; run in image_sims - mode (``-I``) it walks the campaign's tile stores under the run directory - (``tiles///output/run_sp_tile_Mc``), selecting the columns in - ShapePipe's image-sims ``final_cat.param``. + The branch-root fallback is kept for a catalogue merged by hand or by an + older campaign. NOTHING PRODUCES IT any more, so a sim with neither copy + fails at DAG build on a missing input instead of silently re-merging -- + the honest outcome, since the campaign that should have published it did + not. """ - input: - record=f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", - output: - cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - container: - SIF - shell: - "cd {params.run_dir} && " - "{_ENV_PREFIX} python {CREATE_FINAL_CAT} " - "-I -m final_cat_{wildcards.sim}.hdf5 -i .. " - "-p cfis/final_cat.param -P {wildcards.sim} " - "-o n_tiles_final.txt -v" + prod = f"{GRIDS_BASE}/{wc.sim}/product/final_cat_{wc.sim}.hdf5" + if os.path.exists(prod): + return prod + return f"{GRIDS_BASE}/{wc.sim}/final_cat_{wc.sim}.hdf5" rule im_extract: @@ -411,17 +414,30 @@ rule im_extract: ``extract_info.py`` reads ``params.py`` from cwd and the merged catalogue, writing ``shape_catalog_comprehensive_{shape}``. + + Stages the catalogue itself rather than assuming a producer left it in the + right place: ``params_im_sim.py`` sets ``data_dir = "."`` and names + ``final_cat_{name}.hdf5`` with ``name`` from the directory basename, so the + file has to be reachable from the branch dir under exactly that name. A + catalogue published under ``product/`` is linked in; one already at the + root is left alone. """ input: - cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", + cat=merged_cat, params=f"{GRIDS_BASE}/{{sim}}/params.py", output: cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + staged=lambda wc: f"{GRIDS_BASE}/{wc.sim}/final_cat_{wc.sim}.hdf5", container: SIF shell: + # Compare resolved paths: when the catalogue is already at the branch + # root, source and destination are the same file and `ln -sf` would + # replace it with a link to itself. + '[ "$(readlink -f {input.cat})" = "$(readlink -f {params.staged})" ] ' + "|| ln -sf {input.cat} {params.staged}; " "cd {params.run_dir} && {_ENV_PREFIX} python {EXTRACT_INFO}" @@ -446,6 +462,33 @@ rule im_calibrate: "{_ENV_PREFIX} python {CALIBRATE} -s calibrate" +def campaign_records(): + """Per-branch evidence that the ShapePipe campaign finished, for ordering. + + WHICH file that is depends on which workflow ran the campaign, and both + shapes are in use: ShapePipe's unified workflow publishes + product/index/run_report.json, while the older sp_validation-driven runs + left n_tiles_final.txt beside the merged catalogue at the branch root. + Pick whichever exists, so a grid processed years apart still gets an edge. + + Nothing READS these -- im_mbias_config.py takes no input.* -- so the path + only has to name a file that really exists for that branch. A branch with + none contributes nothing rather than blocking the DAG on a file no rule + produces; its merged catalogue is already a declared input elsewhere. + """ + found = [] + for sim in SIMS: + for cand in ( + f"{GRIDS_BASE}/{sim}/product/index/run_report.json", + f"{GRIDS_BASE}/{sim}/product/n_tiles_final.txt", + f"{GRIDS_BASE}/{sim}/n_tiles_final.txt", + ): + if os.path.exists(cand): + found.append(cand) + break + return found + + rule im_mbias_config: """Assemble ``m_bias_config.yaml`` for the m-bias step: the manifest's shear/branch facts, this run's science knobs, and git/container provenance. @@ -455,10 +498,14 @@ rule im_mbias_config: cats=expand( f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS ), - # The per-branch ShapePipe campaign records, for provenance. - campaigns=expand( - f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", sim=SIMS - ), + # The per-branch ShapePipe campaign records, for provenance: the + # edge that says every campaign finished before this config was + # written. ShapePipe's own workflow publishes run_report.json with the + # products; the old logs/shapepipe_campaign.yaml was im_pipeline's + # output and is never written now that `sp run` drives the campaigns. + # Nothing READS these -- im_mbias_config.py takes no input.* -- so the + # path only has to name something the campaign really produced. + campaigns=campaign_records(), output: cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", params: @@ -477,8 +524,15 @@ rule im_mbias_config: n_bootstrap=IMSIM["n_bootstrap"], pair_match=IMSIM["pair_match"], bootstrap_seed=IMSIM["bootstrap_seed"], - container: - SIF + # NO container: -- deliberately. This is snakemake's `script:` directive, + # which pickles the `snakemake` object on the host and unpickles it inside + # the job, so host and job need compatible snakemake versions. They are not: + # the host has 9.16.3 and the image 9.22.0, and the pickle fails with + # "Can't get attribute 'InputFiles' on module snakemake.io". Nothing here + # needs the image anyway -- the script imports only hashlib, os, re, + # subprocess and yaml, and reads git metadata from the checkouts, which is + # easier outside the container than in. The `shell:` rules keep theirs; + # they pass strings, not pickles. script: "../scripts/im_mbias_config.py" From ba4c8d002686d90c70d2c9d4a685953ce102ee9f Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 2 Oct 2026 11:54:35 +0200 Subject: [PATCH 21/36] image sims: im_init copies the mask config instead of linking it config_mask.yaml was a symlink into the sp_validation checkout the run came from. Removing that checkout left every run dir's link dangling, and the next launch failed at DAG time ("config_mask.yaml seems to be a broken symlink", evaluating im_calibrate's mem_mb from its input sizes). With spv the source is the code snapshot, which each launch refreshes. A copy keeps the mask the run dir was calibrated with. Co-Authored-By: Claude Opus 5.5 --- workflow/rules/image_sims.smk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 4d088470..682e21fd 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -330,8 +330,11 @@ rule im_init: ``cfis`` link to ShapePipe's image-sims config dir. ``params_im_sim.py`` derives the field name from the directory basename, so - the same template serves every sim; ``config_mask.yaml`` and ``cfis`` are - symlinks the downstream calibration and merge steps read from cwd. + the same template serves every sim; ``config_mask.yaml`` (a copy) and + ``cfis`` (a symlink) are what the downstream calibration and merge steps + read from cwd. The mask config is copied, not linked: a link into the + sp_validation checkout or spv's code snapshot breaks when that is removed + or refreshed, and the run dir should keep the mask it was calibrated with. ``tiles_{sim}.txt`` is both the ShapePipe campaign's ``tile_list`` and the tile-ID file ``params.py`` names for the found/missing-tile check. """ @@ -355,7 +358,7 @@ rule im_init: # accept for a symlink to a directory. "mkdir -p $(dirname {output.params}) && " "cp {input.template} {output.params} && " - "ln -sf {input.mask_src} {output.mask} && " + "rm -f {output.mask} && cp {input.mask_src} {output.mask} && " "ln -sfT {params.config_dir} {params.cfis} && " "printf '%s\\n' {params.tile_ids} > {output.tiles}" From 528df35d4bbce8f65f719c38ca2ee68cef2f5d8d Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sun, 4 Oct 2026 11:07:40 +0200 Subject: [PATCH 22/36] Exclude n34 from candide jobs n34 hit the per-user thread limit during the image-sims runs, as did n35. Co-Authored-By: Claude Opus 5.5 --- workflow/profiles/candide/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 5415caf5..009dccdf 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -57,7 +57,7 @@ default-resources: slurm_partition: "comp,pscomp" runtime: "60m" cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n35,n36'" + slurm_extra: "'--exclude=n17,n34,n35,n36'" # Snakemake refuses a real run on a remote executor without a job bound. jobs: 100 From 766cc568c365642ab188fd0fbb6214d77e24a367 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 5 Oct 2026 07:00:03 +0200 Subject: [PATCH 23/36] image_sims: accept exp_num again The merge of the uncommitted imsims_workflow work (b268fd6) dropped exp_num from the recognised keys while EXP_NUM still reads it and config.yaml still documents it for the star sims (tiles from *_N, exposures from *_5), so a star-sim run config naming it died at parse as an unknown key. Co-Authored-By: Claude Opus 5.5 --- workflow/rules/image_sims.smk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 682e21fd..2b61b2b8 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -69,6 +69,9 @@ _OPTIONAL_KEYS = { # `config:` sets the cluster's image -- a machine-specific path belongs # with the machine, not in this shared file. "sif", + # Exposures from another realization than the tiles (non-grid sims only; + # default: num). See EXP_NUM below and config.yaml. + "exp_num", } # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { From 00766f9ff5349ff5829ec0e6ed7da9a3f98a7ef1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 26 Oct 2026 19:51:17 +0100 Subject: [PATCH 24/36] Unify metacal weights, response and simulation calibration products --- config/calibration/mask_v2.0.yaml | 6 +- .../calibrate_comprehensive_cat.py | 37 +- scripts/calibration/extract_info.py | 758 +----------------- scripts/calibration/params.py | 11 +- scripts/calibration/stage_image_sim_config.py | 29 + .../examples/demo_calibrate_minimal_cat.py | 14 +- ...ration+leakage_correction_ellipticities.py | 114 +-- src/sp_validation/calibration.py | 485 +++++++++-- src/sp_validation/catalog.py | 8 +- src/sp_validation/catalog_builders.py | 20 +- .../cosmo_val/catalog_characterization.py | 4 +- src/sp_validation/cosmo_val/core.py | 12 +- src/sp_validation/des_jackknife.py | 189 +++++ src/sp_validation/image_sim_config.py | 33 + src/sp_validation/image_sims.py | 189 ++++- src/sp_validation/io.py | 27 +- src/sp_validation/response.py | 188 +++++ src/sp_validation/response_diagnostics.py | 189 +++++ src/sp_validation/rho_tau.py | 1 + workflow/image_sims/config.yaml | 30 +- workflow/rules/image_sims.smk | 38 +- 21 files changed, 1391 insertions(+), 991 deletions(-) create mode 100644 scripts/calibration/stage_image_sim_config.py create mode 100644 src/sp_validation/des_jackknife.py create mode 100644 src/sp_validation/image_sim_config.py create mode 100644 src/sp_validation/response.py create mode 100644 src/sp_validation/response_diagnostics.py diff --git a/config/calibration/mask_v2.0.yaml b/config/calibration/mask_v2.0.yaml index cb89336b..f1f0ea73 100644 --- a/config/calibration/mask_v2.0.yaml +++ b/config/calibration/mask_v2.0.yaml @@ -146,5 +146,7 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False - # Weight for global response matrix, None for unweighted mean - global_R_weight: w + # DES case d: one fixed no-shear W, evaluated on each branch's SNR/size. + weight_scheme: des + w_col: w_des + des_num_bins: 20 diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index c5248c3e..76e6fcef 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -25,7 +25,6 @@ import sp_validation.catalog as cat from sp_validation import calibration from sp_validation import catalog_builders as sp_joint -from sp_validation.calibration import metacal # %% # Initialize calibration class instance @@ -81,17 +80,8 @@ cm = config["metacal"] # %% -gal_metacal = metacal( - dat, - mask_combined._mask, - snr_min=cm["gal_snr_min"], - snr_max=cm["gal_snr_max"], - rel_size_min=cm["gal_rel_size_min"], - rel_size_max=cm["gal_rel_size_max"], - size_corr_ell=cm["gal_size_corr_ell"], - sigma_eps=cm["sigma_eps_prior"], - global_R_weight=cm["global_R_weight"], - verbose=True, +gal_metacal = calibration.metacal_from_config( + dat, mask_combined._mask, cm, verbose=True ) # %% @@ -176,7 +166,6 @@ # %% # Additional data columns to write to output cat add_cols = [ - "w_iv", "FLUX_RADIUS", "MAGERR_AUTO", "NGMIX_T_NOSHEAR", @@ -188,6 +177,16 @@ add_cols_data[key] = cat.get_col(dat, key, mask_combined._mask, mask_metacal) # %% +# Never copy stale unsquared IV weights from a comprehensive input. +if gal_metacal.weight_scheme != "iv": + add_cols_data["w_iv"] = gal_metacal.ns["w_iv"][mask_metacal] +add_cols_format = {} +if "TILE_ID" in dat.dtype.names: + add_cols_data["TILE_ID"] = np.asarray( + cat.get_col(dat, "TILE_ID", mask_combined._mask, mask_metacal) + ).astype(str) + add_cols_format["TILE_ID"] = f"{max(map(len, add_cols_data['TILE_ID']))}A" + # Additional post-processing columns to write to output cat add_cols_post = [ "R_g11", @@ -227,7 +226,7 @@ output_shape_cat_path, ra, dec, - cat_gal["w_des"], + cat_gal["w_des"] if gal_metacal._legacy_response else w, mag=mag, snr=cat_gal["snr"], g=g_corr_mc, @@ -238,12 +237,20 @@ R_select=gal_metacal.R_selection, c=c, c_err=c_err, - w_type="des", + w_type=gal_metacal.weight_scheme or "des", add_cols=add_cols_data, + add_cols_format=add_cols_format, add_header=header, ) # %% +groups = ( + np.asarray(dat["TILE_ID"][mask_combined._mask]) + if "TILE_ID" in dat.dtype.names + else None +) +calibration.save_response_product(output_shape_cat_path, gal_metacal, groups) + # Write mask information to ASCII file with open("masks.txt", "w") as f_out: for my_mask in masks: diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index bfcd1af3..23a11d73 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -33,17 +33,14 @@ import sys import h5py -import numpy as np # from sp_validation.catalog import * from sp_validation import catalog as spv_cat -from sp_validation import galaxy -from sp_validation.calibration import * +from sp_validation import galaxy, response from sp_validation.calibration import metacal -from sp_validation.galaxy import * -from sp_validation.io import * -from sp_validation.io import read_catalogue -from sp_validation.survey import * +from sp_validation.galaxy import T_to_fwhm, classification_galaxy_overlap_ra_dec +from sp_validation.io import open_stats_file, print_stats, read_catalogue +from sp_validation.survey import get_area, missing_tiles # ## 1. Set-up @@ -239,6 +236,11 @@ for key in add_cols_pre_cal: ext_cols_pre_cal[key] = dd[key] +# Retain upstream branch errors when supplied, even with older params files. +# Historical HDF5 merges omitted them; that must not block DES extraction. +# Modern IV rejects an incomplete set at calibration, rather than freezing W. +ext_cols_pre_cal.update(response.branch_error_columns(dd)) + # Flag to cut duplicate objects in overlapping region with neighbouring tiles ext_cols_pre_cal["overlap"] = cut_overlap add_cols_pre_cal_format["overlap"] = "I" @@ -291,742 +293,8 @@ print("No selection and calibration; exiting here") sys.exit(0) else: - if verbose: - print("Continuing with selection and calibration") - os.makedirs(os.path.join(output_dir, plot_dir), exist_ok=True) - -# ## 4. Select galaxies - -# #### Common flags and cuts -# First, set cuts common to all objects: -# - magnitude: cut galaxies that are too faint (= too noisy, likely to be -# artefacts), and too bright (might be too large for postage stamp) -# - flags: cut objects that were flagged as invalid or masked -# - n_epoch: select objects observed on at leatst one epoch (for safety, -# to avoid potential errors with empty data) - -# + -cut_common = classification_galaxy_base( - dd, - cut_overlap, - gal_mag_bright=gal_mag_bright, - gal_mag_faint=gal_mag_faint, - flags_keep=flags_keep, - n_epoch_min=n_epoch_min, - mask_columns=mask_columns, -) -if shape == "ngmix": - m_gal = classification_galaxy_ngmix( - dd, - cut_common, - stats_file, - verbose=verbose, + raise ValueError( + "Direct extraction calibration has no shared weight_scheme/selection " + "configuration. Run calibrate_comprehensive_cat.py with mask_v2.0.yaml " + "on the comprehensive five-branch product instead." ) -else: - raise ValueError(f"Invalid shape measurement method {shape}") - -n_ok = sum(cut_common) -print_stats( - f"objects after common cut: {n_ok:10d}, {n_ok / n_obj:3.2%}", - stats_file, - verbose=verbose, -) - -# MKDEBUG for debugging calibrate_comprehensive -n_ok = sum(m_gal) -print_stats( - f"common & ngmix = galaxy selection: {n_ok:10d}, {n_ok / n_obj:3.2%}", - stats_file, - verbose=verbose, -) -# - - -# ### Metacal global - -# + -import os - -from cs_util.plots import plot_histograms -from lenspack.geometry.projections.gnom import radec2xy -from uncertainties import ufloat - -from sp_validation.calibration import * -from sp_validation.calibration import metacal -from sp_validation.format import * -from sp_validation.plot_style import * -from sp_validation.plots import * - -# - -from sp_validation.survey import * - -# ## metacalibration for galaxies - -# + -gal_metacal = metacal( - dd, - m_gal, - snr_min=gal_snr_min, - snr_max=gal_snr_max, - rel_size_min=gal_rel_size_min, - rel_size_max=gal_rel_size_max, - size_corr_ell=gal_size_corr_ell, - sigma_eps=sigma_eps_prior, - verbose=verbose, -) - -print_stats( - f"Number of objects on metal input = {gal_metacal._n_input}", - stats_file, - verbose=verbose, -) -print_stats( - f"Number of objects after galaxy selection masking = {gal_metacal._n_after_gal_mask}", - stats_file, - verbose=verbose, -) -# - - -# #### Extract quantities after calibration and cuts (in metacal) - -# + -g_corr, g_uncorr, w, mask = get_calibrated_quantities(gal_metacal) - -n_before_cuts = len(gal_metacal.ns["g1"]) -n_after_cuts = len(w) -print_stats( - f"Number of objects before cuts = {n_before_cuts}", stats_file, verbose=verbose -) -print_stats( - f"Number of objects after cuts = {n_after_cuts}", stats_file, verbose=verbose -) - -# coordinates -ra = spv_cat.get_col(dd, col_name_ra, m_gal, mask) -dec = spv_cat.get_col(dd, col_name_dec, m_gal, mask) - -# Modify R.A. for plots if R.A. = 0 in area -if wrap_ra != 0: - ra_wrap = (ra + wrap_ra) % 360 - wrap_ra + 360 -else: - ra_wrap = ra - -ra_mean = np.mean(ra_wrap) -dec_mean = np.mean(dec) - -print_stats( - "Mean coordinates (ra, dec) =" - + f" ({ra_mean:.3f}, {dec_mean:.3f}) deg," - + f" wrap_ra={wrap_ra} deg", - stats_file, - verbose=verbose, -) - -# magnitude, from SExtractor -mag = spv_cat.get_col(dd, "MAG_AUTO", m_gal, mask) - -# Keep tile ID if external mask -if mask_external_path: - tile_ID = spv_cat.get_col(dd, "TILE_ID", m_gal, mask) - -snr = spv_cat.get_snr(shape, dd, m_gal, mask) - -# + -# Add additional columns with metacal mask - -if add_cols: - add_cols_data = {} - for key in add_cols: - add_cols_data[key] = dd[key][m_gal][mask] -else: - add_cols_data = None - -# + -#### Compute coordinates for projection and spatial binning (needed later) -# - - -# Project all objects from spherical to Cartesian coordinates -x, y = radec2xy(ra_mean, dec_mean, ra_wrap, dec) - -# #### Compute field size - -# + -# Define mix, max and size -min_x = np.min(x) -max_x = np.max(x) -min_y = np.min(y) -max_y = np.max(y) - -size_x_deg = np.rad2deg(max_x - min_x) -size_y_deg = np.rad2deg(max_y - min_y) - -print_stats( - "Field size in projected coordinates is (x, y) " - + f"= ({size_x_deg:.2f}, {size_y_deg:.2f}) deg", - stats_file, - verbose=verbose, -) - -# + -# Number density -n_gal = len(w) -n_gal_sm = len(np.where(m_gal)[0]) - -print_stats( - f"Number of galaxies after metacal = {n_gal}/{n_gal_sm} " - + f"= {n_gal / n_gal_sm * 100:.1f}%", - stats_file, - verbose=verbose, -) -print_stats( - f"Galaxy density (ignoring masks+overlaps)= {n_gal / area_amin2:.2f} gal/arcmin2", - stats_file, - verbose=verbose, -) -# - - -# Write number of galaxies for each tile to file -fname = f"{output_dir}/tile_id_gal_counts_{shape}.txt" -detection_IDs = dd["TILE_ID"] -galaxy_IDs = detection_IDs[m_gal] -shape_IDs = galaxy_IDs[mask] -write_tile_id_gal_counts(detection_IDs, galaxy_IDs, shape_IDs, fname) - -# + -# Add all weights (for combining weighted averages of sub-samples) - -w_tot = np.sum(w) - -print_stats(f"Sum of weights = {w_tot:.1f}", stats_file, verbose=verbose) - -# + -# Effective sample size ESS = 1/sum(w_n^2) -# The inverse sum over squared normalised weights -# Range [1; N] - -# normalised weights -wn = w / w_tot -s = np.sum(wn**2) -ess = 1 / s - -print_stats( - f"effective sample size, ESS/N = {ess:.1f}/{n_gal} = {ess / n_gal:.3g}", - stats_file, - verbose=verbose, -) -# - - -# #### Plot spatial distribution of objects - -x_label = "R.A. [deg]" -y_label = "DEC [deg]" -cbar_label_base = "Density [$A_{\\rm pix}^{-1}$]" - -len(dec) - -# + -# Galaxies -Apix = 1 # [arcmin^2] -cbar_label = "{}, $A_{{\\rm pix}} \\approx {:.1g}$ arcmin$^2$".format( - cbar_label_base, Apix -) -n_grid = int(np.sqrt(area_amin2) / Apix) -if verbose: - print("Number of pixels = {}^2".format(n_grid)) - -title = f"Galaxies ({shape})" -out_path = f"{plot_dir}/galaxy_number_count_{shape}" -plot_spatial_density( - ra_wrap, - dec, - title, - x_label, - y_label, - cbar_label, - out_path, - n_grid=n_grid, - verbose=verbose, -) - -# All objects without overlap, useful for position-only -# setting (no shapes) -title = "Galaxies (all, no overlap)" -out_path = f"{plot_dir}/galaxy_number_count_all_nooverlap" -plot_spatial_density( - ra_wrap_all[cut_overlap], - dec_all[cut_overlap], - title, - x_label, - y_label, - cbar_label, - out_path, - n_grid=n_grid, - verbose=verbose, -) -# - - -# #### Plot galaxy signal-to-noise distribution - -# + -x_label = "SNR" -y_label = "Frequency" -density = True -x_range = (0, 200) -n_bin = 500 -x_cut = gal_snr_min - -if shape == "ngmix": - # Do not apply `mask_ns`, so use all galaxies - xs = [dd["NGMIX_FLUX_NOSHEAR"][m_gal] / dd["NGMIX_FLUX_ERR_NOSHEAR"][m_gal]] - labels = ["$F/\\sigma(F)$"] -else: - raise ValueError(f"Unknown shape measurement method {shape}") - -title = "Galaxies" - -out_name = f"hist_SNR_{shape}.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - vline_x=[x_cut], - vline_lab=[f"SNR = {x_cut}"], -) -# - - -# ### Plot galaxy relative size distribution - -# + -x_label = r"$R^2_{\rm gal} / R^2_{\rm PSF}$" -y_label = "Frequency" -density = True -x_range = (0, 1.5) -n_bin = 500 -x_cut = gal_rel_size_min - -labels = [] -if shape == "ngmix": - # Do not apply `mask_ns`, so use all galaxies - xs = [dd["NGMIX_T_NOSHEAR"][m_gal] / dd["NGMIX_T_PSF_RECONV_NOSHEAR"][m_gal]] - labels.append("size ratio") - -else: - raise ValueError(f"Unknown shape measurement method {shape}") - -title = "Galaxies" - -out_name = f"hist_rel_size_{shape}.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - vline_x=[x_cut], - vline_lab=[f"rel_size = {x_cut}"], -) -# - - -# ## Metacalibration for stars - -if star_cat_path: - star_metacal = metacal(dd[ind_star], m_star, masking_type="star", verbose=verbose) - - # #### Number density - - # + - # mask for 'no shear' images - - mask_ns_stars = star_metacal.mask_dict["ns"] - n_star = len(star_metacal.ns["g1"][mask_ns_stars]) - - print_stats(f"Number of stars = {n_star}", stats_file, verbose=verbose) - print_stats( - "Star density = {:.2f} stars/deg2".format(n_star / area_deg2), - stats_file, - verbose=verbose, - ) - # - - -# ## Additive bias -# Use raw, uncorrected ellipticities. - -print_stats("additive bias", stats_file, verbose=verbose) - -# + -# Compute mean, weighted mean, and (Poisson) error - -c = np.zeros(2) -c_err = np.zeros(2) -cw = np.zeros(2) -cw_err = np.zeros(2) - -for comp in (0, 1): - c[comp] = np.average(g_uncorr[comp]) - c_err[comp] = np.std(g_uncorr[comp]) - - cw[comp] = np.average(g_uncorr[comp], weights=w) - variance = np.average((g_uncorr[comp] - cw[comp]) ** 2, weights=w) - cw_err[comp] = np.sqrt(variance) - -for comp in (0, 1): - print_stats(f"c_{comp + 1} = {c[comp]:.3g}", stats_file, verbose=verbose) - print_stats(f"cw_{comp + 1} = {cw[comp]:.3g}", stats_file, verbose=verbose) -for comp in (0, 1): - print_stats(f"dc_{comp + 1} = {c_err[comp]:.3g}", stats_file, verbose=verbose) - print_stats(f"dcw_{comp + 1} = {cw_err[comp]:.3g}", stats_file, verbose=verbose) - -# Error of mean: divide by sqrt(N) (TBC whether this is correct) -for comp in (0, 1): - print_stats( - f"dmc_{comp + 1} = {c_err[comp] / np.sqrt(n_gal):.3e}", - stats_file, - verbose=verbose, - ) - print_stats( - f"dmcw_{comp + 1} = {cw_err[comp] / np.sqrt(n_gal):.3e}", - stats_file, - verbose=verbose, - ) - -# + -# Compute jackknife mean and errors - -remove_size = 0.05 - -cjk = np.zeros(2) * -1 -cjk_err = np.zeros(2) * -1 - -for comp in (0, 1): - if n_jack > 0: - cjk[comp], cjk_err[comp] = jackknif_weighted_average( - g_uncorr[comp], w, remove_size=remove_size, n_realization=n_jack - ) - cjk_dc = ufloat(cjk[comp], cjk_err[comp]) - print_stats(f"cjk_{comp + 1} = {cjk_dc:.3eP}", stats_file, verbose=verbose) -# - - -# ## Get quantities calibrated for both multiplicative and additive bias - -g_corr_mc = np.zeros_like(g_corr) -c_corr = np.linalg.inv(gal_metacal.R).dot(c) -for comp in (0, 1): - g_corr_mc[comp] = g_corr[comp] - c_corr[comp] - -# ## Response matrix - -# ### Mean - -# + -print_stats("total response matrix:", stats_file, verbose=verbose) -rs = np.array2string(gal_metacal.R) -print_stats(rs, stats_file, verbose=verbose) - -print_stats("shear response matrix:", stats_file, verbose=verbose) -R_shear = np.mean(gal_metacal.R_shear, 2) -rs = np.array2string(R_shear) -print_stats(rs, stats_file, verbose=verbose) - -print_stats("selection response matrix:", stats_file, verbose=verbose) -rs = np.array2string(gal_metacal.R_selection) -print_stats(rs, stats_file, verbose=verbose) - -# + -if star_cat_path: - print_stats("stars:", stats_file, verbose=verbose) - - print_stats("total response matrix:", stats_file, verbose=verbose) - rs = np.array2string(star_metacal.R) - print_stats(rs, stats_file, verbose=verbose) - - print_stats("shear response matrix:", stats_file, verbose=verbose) - R_shear_stars = np.mean(star_metacal.R_shear, 2) - rs = np.array2string(R_shear_stars) - print_stats(rs, stats_file, verbose=verbose) - - print_stats("selection response matrix:", stats_file, verbose=verbose) - rs = np.array2string(star_metacal.R_selection) - print_stats(rs, stats_file, verbose=verbose) -# - - -# ### Plot distribution of response matrix elements - -x_label = "response matrix element" -y_label = "Frequency" -x_range = (-3, 3) -n_bin = 500 - -colors = ["blue", "red", "blue", "red"] -linestyles = ["-", "-", ":", ":"] - -# + -labels = ["$R_{11}$ galaxies", "$R_{22}$ galaxies"] -xs = [gal_metacal.R_shear[0, 0], gal_metacal.R_shear[1, 1]] -if star_cat_path: - labels += ["$R_{11}$ stars", "$R_{22}$ stars"] - xs += [star_metacal.R_shear[0, 0], star_metacal.R_shear[1, 1]] -title = shape - -out_name = f"R_{shape}_diag.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - colors=colors[: len(xs)], - linestyles=linestyles[: len(xs)], -) - -# + -labels = ["$R_{12}$ galaxies", "$R_{21}$ galaxies"] -xs = [gal_metacal.R_shear[0, 1], gal_metacal.R_shear[1, 0]] -if star_cat_path: - labels += ["$R_{12}$ stars", "$R_{21}$ stars"] - xs += [star_metacal.R_shear[0, 1], star_metacal.R_shear[1, 0]] -title = shape -out_name = f"R_{shape}_offdiag.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - colors=colors[: len(xs)], - linestyles=linestyles[: len(xs)], -) -# - - -# ## Ellipticities - -# + -x_label = "ellipticity" -y_label = "Frequency" -x_range = (-1, 1) -n_bin = 500 - -labels = ["$e_1$", "$e_2$"] -colors = ["blue", "red"] -linestyles = ["-", "-"] - -# + -xs = [g_corr[0], g_corr[1]] -weights = [w] * 2 - -title = f"{shape} galaxies" -out_name = f"ell_gal_{shape}.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - weights=weights, - colors=colors, - linestyles=linestyles, -) - -# + -if star_cat_path: - xs = [star_metacal.ns["g1"][mask_ns_stars], star_metacal.ns["g2"][mask_ns_stars]] - weights = [star_metacal.ns["w"][mask_ns_stars]] * 2 - - title = "stars" - out_name = f"ell_stars_{shape}.pdf" - out_path = os.path.join(plot_dir, out_name) - - plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - weights=weights, - colors=colors, - linestyles=linestyles, - ) - # - - - x_range = (-0.15, 0.15) - n_bin = 250 - - # + - xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] - title = "PSF" - out_name = f"ell_PSF_{shape}.pdf" - out_path = os.path.join(plot_dir, out_name) - - plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - colors=colors, - linestyles=linestyles, - ) -# - - -# ## Magnitudes - -# + -x_label = "$r$-band magnitude" -y_label = "Frequency" -x_range = (gal_mag_bright + 1, gal_mag_faint - 1) -n_bin = 500 - -colors = ["blue", "red"] -linestyles = ["-", "-"] - -title = "galaxies" -out_name = "mag_gal.pdf" -out_path = os.path.join(plot_dir, out_name) - -# + -labels = [shape] -xs = [dd["MAG_AUTO"][m_gal][mask]] - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - colors=colors, - linestyles=linestyles, -) -# - - -# ## Ellipticity dispersion - -sig_eps = np.sqrt(np.var(g_corr[0]) + np.var(g_corr[1])) -print_stats( - "Dispersion of complex ellipticity = {:.3f}".format(sig_eps), - stats_file, - verbose=verbose, -) -print_stats( - "Dispersion of (average) single-component ellipticity = {:.3f} = {:.3f} / sqrt(2)" - "".format(sig_eps / np.sqrt(2), sig_eps), - stats_file, - verbose=verbose, -) - -# ## Write catalogues - -import os - -from sp_validation.catalog import * -from sp_validation.format import * - -# Shear response per galaxy -R_shear_ind = gal_metacal.R_shear - -# ### Write basic shape catalogue - -spv_cat.write_shape_catalog( - f"{output_shape_cat_stem}_{shape}{output_ext}", - ra, - dec, - w, - mag=mag, - g=g_corr_mc, - g1_uncal=g_uncorr[0], - g2_uncal=g_uncorr[1], - R=gal_metacal.R, - R_shear=R_shear, - R_select=gal_metacal.R_selection, - c=c, - c_err=c_err, - add_cols=add_cols_data, -) - -# ### Write extended shape catalogue - -ext_cols = {} -if add_cols: - ext_cols = add_cols_data -else: - ext_cols = {} - -# Optional: Create flag from external mask -if mask_external_path: - m_extern = mask_overlap(ra, dec, tile_ID, mask_external_path) - -# + -# Additional columns: -# {e1, e2, size}_PSF -ext_cols["e1_PSF"] = dd[key_PSF_g1][m_gal][mask] -ext_cols["e2_PSF"] = dd[key_PSF_g2][m_gal][mask] -ext_cols["fwhm_PSF"] = size_to_fwhm(dd[key_PSF_size][m_gal][mask]) -if mask_external_path: - ext_cols["mask_extern"] = m_extern - -# Extended catalogue with SNR, individual R matrices, ext_cols -spv_cat.write_shape_catalog( - f"{output_shape_cat_stem}_extended_{shape}{output_ext}", - ra, - dec, - w, - mag=mag, - snr=snr, - g=g_corr_mc, - g1_uncal=g_uncorr[0], - g2_uncal=g_uncorr[1], - R_g11=R_shear_ind[0, 0], - R_g12=R_shear_ind[0, 1], - R_g21=R_shear_ind[1, 0], - R_g22=R_shear_ind[1, 1], - R=gal_metacal.R, - R_shear=R_shear, - R_select=gal_metacal.R_selection, - c=c, - c_err=c_err, - add_cols=ext_cols, -) -# - - -# ### Write galaxy (or random) position catalogue - -if shape == "": - print("writing random cat (hack)") - - ra = dd["RA"][cut_overlap] - dec = dd["DEC"][cut_overlap] - tile_id = dd["TILE_ID"][cut_overlap] - write_galaxy_cat(f"{output_shape_cat_stem}{output_ext}", ra, dec, tile_id) diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index 63dc0828..6dcc6bd9 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -178,7 +178,16 @@ # Create key names for metacal information prefix = "NGMIX" suffixes = ["1M", "1P", "2M", "2P", "NOSHEAR"] -centers = ["FLAGS", "G1", "G2", "FLUX", "FLUX_ERR", "T", "T_ERR", "T_PSF_RECONV"] +centers = [ + "FLAGS", + "G1", + "G2", + "FLUX", + "FLUX_ERR", + "T", + "T_ERR", + "T_PSF_RECONV", +] for center in centers: for suffix in suffixes: add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") diff --git a/scripts/calibration/stage_image_sim_config.py b/scripts/calibration/stage_image_sim_config.py new file mode 100644 index 00000000..83eda5ed --- /dev/null +++ b/scripts/calibration/stage_image_sim_config.py @@ -0,0 +1,29 @@ +"""Stage a simulation mask config with the independent DES-table policy.""" + +import argparse +from pathlib import Path + +import yaml + +from sp_validation.image_sim_config import calibration_config + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--input", type=Path, required=True) +p.add_argument("--output", type=Path, required=True) +p.add_argument("--w-cols", nargs="+", required=True) +p.add_argument("--source", default="external") +p.add_argument("--product-name", default="") +p.add_argument("--product-path", default="") +a = p.parse_args() +config = calibration_config( + yaml.safe_load(a.input.read_text()), + a.w_cols, + a.source, + dict(name=a.product_name, path=a.product_path) + if a.product_name not in ("", "-") and a.product_path not in ("", "-") + else None, +) +# An earlier workflow may have staged a symlink to a shared mask config. +# Replace that symlink rather than overwriting its target. +a.output.unlink(missing_ok=True) +a.output.write_text(yaml.safe_dump(config, sort_keys=False)) diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 50e52657..87ff13d1 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -22,7 +22,7 @@ import sp_validation.catalog as cat from sp_validation import catalog_builders as sp_joint -from sp_validation.calibration import get_calibrated_m_c, metacal +from sp_validation.calibration import get_calibrated_m_c, metacal_from_config # Initialize calibration class instance obj = sp_joint.CalibrateCat() @@ -95,17 +95,7 @@ cm = config["metacal"] -gal_metacal = metacal( - dat, - mask_combined._mask, - snr_min=cm["gal_snr_min"], - snr_max=cm["gal_snr_max"], - rel_size_min=cm["gal_rel_size_min"], - rel_size_max=cm["gal_rel_size_max"], - size_corr_ell=cm["gal_size_corr_ell"], - sigma_eps=cm["sigma_eps_prior"], - verbose=True, -) +gal_metacal = metacal_from_config(dat, mask_combined._mask, cm, verbose=True) # - g_corr_mc, g_uncorr, w, mask_metacal, c, c_err = get_calibrated_m_c(gal_metacal) diff --git a/scripts/shear_calibration+leakage_correction_ellipticities.py b/scripts/shear_calibration+leakage_correction_ellipticities.py index 7d040052..b3fa2383 100644 --- a/scripts/shear_calibration+leakage_correction_ellipticities.py +++ b/scripts/shear_calibration+leakage_correction_ellipticities.py @@ -1,110 +1,22 @@ -# Name file: shear_calibration+leakage_correction_ellipticities_.py +"""The matched-cut recalibration recipe is scientifically incomplete. -# @author: Antonin Corinaldi, antonin.corinaldi@cea.fr - -# Python file to perform metacalibration and PSF leakage correction of the ellipticities of matched catalogues -# (cf file get_matched_catalogue.py in src/sp_validation for the creation of the matched catalogues) - - -import numpy as np -from astropy.table import Table -from calibration import fill_cat_gal, get_alpha_leakage_per_object - - -class ResponseMatrix: - def __init__(self, R11, R12, R21, R22): - self.R11 = R11 - self.R12 = R12 - self.R21 = R21 - self.R22 = R22 - - -def get_R(cat, weights=None): - """ - Compute weighted mean response matrix. - """ - R = np.zeros((2, 2)) - - for i in range(2): - for j in range(2): - R[i, j] = np.average(cat[f"R_g{i + 1}{j + 1}"], weights=weights) - - return R +A matched cut catalogue contains only R_g and cannot reconstruct the selection +and weight response of the new population. Calibrate from aligned parent +five-branch inputs via sp_validation.calibration.metacal_from_config instead. +PSF leakage and additive bias are distinct corrections. +""" def process_matched_catalogue( input_path, output_path=None, weight_column="w_iv", num_bins=20, weight_type="iv" ): - """ - Function to: - - read matched catalogue - - compute metacalibration correction - - apply PSF leakage correction - - save corrected matched catalogue - - Parameters - ---------- - input_path : str - Path to input FITS catalogue - output_path : str or None - Path to save output catalogue (if None, overwrite input) - weight_column : str - Column used for weighting (default: w_iv) - num_bins : int - Number of bins for leakage estimation - weight_type : str - Weight type for alpha leakage computation - """ - - # Reading catalogue - table = Table.read(input_path) - - names = [name for name in table.colnames if len(table[name].shape) <= 1] - table = table[names] - - df = table.to_pandas() - - # Computing response matrix - g_uncorr = np.array([df["e1_uncal"], df["e2_uncal"]]) - - gal_metacal = ResponseMatrix(df["R_g11"], df["R_g12"], df["R_g21"], df["R_g22"]) - - cat = {} - fill_cat_gal( - cat, df, g_uncorr, gal_metacal, mask1=None, mask2=None, purpose="weights" - ) - - weights = df[weight_column] - R = get_R(cat, weights=weights) - - # Metacalibration correction - R_inv = np.linalg.inv(R) - e_cal = R_inv.dot(np.array(df[["e1_uncal", "e2_uncal"]].T)) - - df["e1"] = e_cal[0, :] - df["e2"] = e_cal[1, :] - - # PSF leakage correction - alpha_1, alpha_2 = get_alpha_leakage_per_object( - df, num_bins=num_bins, weight_type=weight_type + """Refuse to invent the missing selection response of a matched sample.""" + raise ValueError( + "A matched cut catalogue lacks the five-branch selection/weight " + "response for its changed population. Calibrate the parent with " + "metacal_from_config before using a matched sample diagnostically." ) - df["e1_leak_corrected"] = df["e1"] - alpha_1 * df["e1_PSF"] - df["e2_leak_corrected"] = df["e2"] - alpha_2 * df["e2_PSF"] - - # Save catalogue - output_path = output_path or input_path - - output_table = Table.from_pandas(df) - output_table.write(output_path, overwrite=True) - - print(f"Catalogue processed and saved to: {output_path}") - - return output_table - - -# Example of usage -process_matched_catalogue( - "/n17data/corinaldi/matched_catalogues/unions1.6.9/UNIONS1.6.9_lrg_cross.fits" -) +if __name__ == "__main__": + process_matched_catalogue(None) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index b594b9b6..9763fb14 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -8,6 +8,8 @@ """ +import warnings + import numpy as np import pandas as pd import statsmodels.api as sm @@ -16,10 +18,162 @@ from shear_psf_leakage import leakage, run_object from sp_validation import catalog as sp_cat +from sp_validation import response from sp_validation.io import read_catalogue from sp_validation.statistics import jackknif_weighted_average2 +def metacal_from_config(data, mask, cm, verbose=False): + """One calibration entry point; old release configs are explicitly legacy. + + Missing weight_scheme preserves their historical R/c/unsquared IV policy. + New production configurations must declare des or iv and matching w_col. + """ + modern = "weight_scheme" in cm + return metacal( + data, + mask, + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + rel_size_min=cm["gal_rel_size_min"], + rel_size_max=cm["gal_rel_size_max"], + size_corr_ell=cm["gal_size_corr_ell"], + sigma_eps=cm["sigma_eps_prior"], + global_R_weight=None if modern else cm.get("global_R_weight"), + weight_scheme=cm.get("weight_scheme"), + legacy_response=not modern, + des_num_bins=cm.get("des_num_bins", 20), + des_min_count=cm.get("des_min_count", 3), + des_weight_product=cm.get("des_weight_product"), + w_col=cm.get("w_col"), + verbose=verbose, + ) + + +def load_des_weight_product(product): + """Read a named, independent calibration product, never refitting its W. + + Use modern response products, whose headers declare the coordinate + definitions. Historical released weights alone do not specify such a table. + The product's edges and pooling rule are authoritative, not the sim cuts. + """ + if ( + not isinstance(product, dict) + or not product.get("name") + or not product.get("path") + ): + raise ValueError("des_weight_product needs a name and a FITS path") + with fits.open(product["path"]) as hdus: + hdr = hdus[0].header + if ( + hdr.get("WGHTSCH") != "des" + or hdr.get("WSNRCOL") != "FLUX/FLUX_ERR" + or hdr.get("WSIZDEF") != "T/T_PSF_RECONV" + ): + raise ValueError( + "Incompatible DES product: regenerate with coordinate metadata" + ) + es, er = np.array(hdus["W_EDGES"].data, dtype=float) + w = np.array(hdus["W_DES"].data, dtype=float) + count = np.array(hdus["W_COUNT"].data) + if ( + w.shape != (len(es) - 1, len(er) - 1) + or count.shape != w.shape + or not np.all(np.diff(es) > 0) + or not np.all(np.diff(er) > 0) + or not np.isfinite(w).all() + or np.any(w < 0) + or not np.any(w > 0) + ): + raise ValueError("Invalid DES table or edges in calibration product") + return (es, er, w, count), hdr["DESNMIN"] + + +def save_response_product(path, gm, groups=None): + """Attach matrix definitions, fitted W, and spatial response/c covariance. + + groups are aligned to the preselected parent, never only the ns selection. + """ + if gm._legacy_response: + with fits.open(path, mode="update") as hdus: + hdus[0].header["RLEGACY"] = (True, "Historical R/c and unsquared IV policy") + return + with fits.open(path, mode="update") as hdus: + hdr = hdus[0].header + hdr["MEANSUB"] = getattr(gm, "subtract_mean_shear", False) + hdr["WGHTSCH"] = gm.weight_scheme + hdr["RAPPLIED"] = True + hdr["MCSTEP"] = gm._step + hdr["WPOLICY"] = ( + "fixed-external" + if gm._des_weight_product is not None + else "refit" + if gm.weight_scheme == "des" + else "not applicable" + ) + hdr["IVPOLICY"] = "branch variances" + hdr["CMUNITS"] = "raw ellipticity" + for i in range(2): + hdr[f"CE_{i + 1}"] = gm.c_e[i] + hdr[f"CG_{i + 1}"] = gm.c_g[i] + for j in range(2): + hdr[f"RJ_{i + 1}{j + 1}"] = gm.R_joint[i, j] + for b in response.BRANCHES: + hdr[f"N_{b}"] = gm.branch_counts[b] + hdr[f"SW_{b}"] = gm.branch_weight_sums[b] + if hasattr(gm, "branch_migration"): + hdr[f"MIG_{b}"] = gm.branch_migration[b] + if hasattr(gm, "W"): + hdr["DESNMIN"] = (gm._des_min_count, "Minimum DES training count") + hdr["DESMERGE"] = "concentric square; pooled moments" + hdr["WSNRCOL"] = "FLUX/FLUX_ERR" + hdr["WSIZDEF"] = "T/T_PSF_RECONV" + hdr["WSOURCE"] = "external" if gm._des_weight_product else "self" + if gm._des_weight_product: + hdr["WPRODUCT"] = gm._des_weight_product["name"] + hdr["WPRODPTH"] = gm._des_weight_product["path"] + es, er, w, count = gm.W + hdus.append(fits.ImageHDU(w, name="W_DES")) + hdus.append(fits.ImageHDU(count, name="W_COUNT")) + hdus.append(fits.ImageHDU(np.array([es, er]), name="W_EDGES")) + if groups is not None: + labels, estimates, cov = gm.response_jackknife( + groups, refit_w=gm.weight_scheme == "des" + ) + labels = np.asarray(labels).astype(str) + columns = [ + fits.Column( + name="TILE_ID", array=labels, format=f"{max(map(len, labels))}A" + ) + ] + for key, prefix in [ + ("total", "R"), + ("joint", "RJ"), + ("shear", "RG"), + ("selection", "RS"), + ]: + for i in range(2): + for j in range(2): + columns.append( + fits.Column( + name=f"{prefix}_{i + 1}{j + 1}", + array=estimates[key][:, i, j], + format="D", + ) + ) + for key in ("c_e", "c_g"): + for i in range(2): + columns.append( + fits.Column( + name=f"{key}{i + 1}", array=estimates[key][:, i], format="D" + ) + ) + hdus.append(fits.BinTableHDU.from_columns(columns, name="R_JK")) + cov_hdu = fits.ImageHDU(cov, name="R_COV") + cov_hdu.header["ORDER"] = "shear,selection,total,joint,c_e,c_g; row-major" + hdus.append(cov_hdu) + + def get_calibrated_quantities(gal_metacal): """Get Calibrated Quantities. @@ -98,11 +252,18 @@ def get_calibrated_m_c(gal_metacal, additive_correction=True): c_err = np.zeros(2) for comp in (0, 1): - c[comp] = np.mean(g_uncorr[comp]) + if getattr(gal_metacal, "_legacy_response", False): + c[comp] = np.mean(g_uncorr[comp]) + c_err[comp] = np.std(g_uncorr[comp]) + else: + c[comp] = np.average(g_uncorr[comp], weights=w) + c_err[comp] = ( + np.sqrt(np.sum(w**2 * (g_uncorr[comp] - c[comp]) ** 2)) / w.sum() + ) - # MKDEBUG TODO: Use std of mean instead, - # which is consistent with jackknife - c_err[comp] = np.std(g_uncorr[comp]) + gal_metacal.subtract_mean_shear = additive_correction + gal_metacal.c_e = c + gal_metacal.c_g = np.linalg.solve(gal_metacal.R, c) # Shear estimate corrected for additive bias g_corr_mc = np.copy(g_corr) @@ -176,7 +337,6 @@ def cut_to_bins(df, key, num_bins, type="log", x_min=None, x_max=None): def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="weights"): - cat_gal["e1_uncal"] = g_uncorr[0] cat_gal["e2_uncal"] = g_uncorr[1] cat_gal["R_g11"] = gal_metacal.R11 @@ -261,46 +421,33 @@ def get_w_des( DES weights """ - df_gal = build_df(cat_gal) - - # Create logarithmic bins in size and SNR - cut_to_bins( - df_gal, - "snr", - num_bins, - type="log", - x_min=snr_min, - x_max=snr_max, - ) - cut_to_bins( - df_gal, - "size_ratio", - num_bins, - type="log", - x_min=size_ratio_min, - x_max=size_ratio_max, + table = fit_w_des( + cat_gal, num_bins, snr_min, snr_max, size_ratio_min, size_ratio_max ) + return response.evaluate_w(table, cat_gal["snr"], cat_gal["size_ratio"]) - # Compute shape noise and the shear response in each bin - for i in range(num_bins): - for j in range(num_bins): - bin_mask = (df_gal["snr_log_bins"] == i) & ( - df_gal["size_ratio_log_bins"] == j - ) - ngal = np.sum(bin_mask) - if ngal == 0: - print(f"Zero galaxies in snr/size_ratio bin ({i},{j})") - shape_noise = 0.5 * ( - np.sum(df_gal[bin_mask]["e1_uncal"] ** 2) / ngal - + np.sum(df_gal[bin_mask]["e2_uncal"] ** 2) / ngal - ) - response = 0.5 * ( - np.average(df_gal[bin_mask]["R_g11"]) - + np.average(df_gal[bin_mask]["R_g22"]) - ) - df_gal.loc[bin_mask, "w_des"] = response**2 / shape_noise - return np.array(df_gal["w_des"]) +def fit_w_des( + cat_gal, + num_bins=20, + snr_min=None, + snr_max=None, + size_ratio_min=None, + size_ratio_max=None, + min_count=0, +): + """Fit once on raw no-shear shapes; evaluate via response.evaluate_w.""" + return response.fit_w( + np.asarray(cat_gal["snr"]), + np.asarray(cat_gal["size_ratio"]), + np.array([cat_gal["e1_uncal"], cat_gal["e2_uncal"]]), + 0.5 * (np.asarray(cat_gal["R_g11"]) + np.asarray(cat_gal["R_g22"])), + create_bins(cat_gal["snr"], num_bins, x_min=snr_min, x_max=snr_max), + create_bins( + cat_gal["size_ratio"], num_bins, x_min=size_ratio_min, x_max=size_ratio_max + ), + min_count=min_count, + ) def get_alpha_leakage_per_object(cat_gal, num_bins, weight_type="des"): @@ -522,7 +669,6 @@ def get_quantities_binned( which=["response", "number", "leakage"], verbose=True, ): - if verbose: print("Compute binned quantities") @@ -627,6 +773,23 @@ def get_calibrate_e_from_cat(path_cat_gal, weight_type="des", verbose=False): ] ) + stored_scheme = hdu_gal[0].header.get("WGHTSCH") + if stored_scheme != weight_type: + hdu_gal.close() + raise ValueError( + "Selection response cannot be reused with unknown/mismatched " + "weights; recalibrate from the parent five-branch catalogue" + ) + hdr = hdu_gal[0].header + weights = cat_gal["w_" + weight_type] + if hdr.get("N_ns") != len(cat_gal) or not np.isclose( + hdr.get("SW_ns", np.nan), weights.sum(), rtol=1e-12 + ): + hdu_gal.close() + raise ValueError( + "Stored selection response does not describe this " + "population/weights; use the parent five-branch catalogue" + ) if verbose: print("R_select\n", R_select) @@ -640,7 +803,9 @@ def get_calibrate_e_from_cat(path_cat_gal, weight_type="des", verbose=False): R_shear = np.zeros((2, 2)) for iidx in range(2): for jidx in range(2): - R_shear[iidx, jidx] = np.mean(cat_gal[f"R_g{iidx + 1}{jidx + 1}"]) + R_shear[iidx, jidx] = np.average( + cat_gal[f"R_g{iidx + 1}{jidx + 1}"], weights=cat_gal["w_" + weight_type] + ) if verbose: print("R_shear\n", R_shear) @@ -653,9 +818,11 @@ def get_calibrate_e_from_cat(path_cat_gal, weight_type="des", verbose=False): c_corr = np.linalg.inv(R) @ c g_cal = np.linalg.inv(R) @ g - for comp in (0, 1): - g_cal[comp] -= c_corr[comp] + if hdu_gal[0].header.get("MEANSUB", True): + for comp in (0, 1): + g_cal[comp] -= c_corr[comp] + hdu_gal.close() return g_cal[0], g_cal[1] @@ -741,8 +908,38 @@ def __init__( global_R_weight=None, sigma_eps=0.34, verbose=False, + weight_scheme=None, + legacy_response=False, + des_num_bins=20, + des_min_count=3, + des_weight_product=None, + w_col=None, ): - + if weight_scheme is not None and global_R_weight is not None: + raise ValueError("Use weight_scheme, not both response weight knobs") + if weight_scheme is None and not legacy_response: + if global_R_weight == "w": + warnings.warn( + "global_R_weight=w is deprecated; use weight_scheme=iv", + DeprecationWarning, + stacklevel=2, + ) + weight_scheme = "iv" + elif global_R_weight not in (None, "None"): + raise ValueError("Use weight_scheme=des or iv") + else: + legacy_response = True # Historical unweighted diagnostic API. + self.weight_scheme = weight_scheme + self._legacy_response = legacy_response + self._des_num_bins = des_num_bins + self._des_min_count = des_min_count + self._des_weight_product = des_weight_product + if des_weight_product is not None: + if weight_scheme != "des" or legacy_response: + raise ValueError("des_weight_product requires modern weight_scheme=des") + self.W, self._des_min_count = load_des_weight_product(des_weight_product) + if w_col is not None: + response.validate_weight_scheme(weight_scheme, w_col) self._masking_type = masking_type self._step = step @@ -819,7 +1016,20 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): masked_data, self._sigma_eps, mask=None, + legacy=self._legacy_response, ) + ns["w_iv"] = ns["w"] + if self.weight_scheme == "iv" and not self._legacy_response: + for name, branch in zip(("1M", "1P", "2M", "2P"), (m1, p1, m2, p2)): + required = [f"NGMIX_G{i}_ERR_{name}" for i in (1, 2)] + if not all(key in masked_data.dtype.names for key in required): + raise ValueError( + "IV response needs per-branch G1_ERR/G2_ERR; regenerate " + "the comprehensive catalogue from the tile final catalogues" + ) + branch["w_iv"] = self.get_variance_ivweights( + masked_data, self._sigma_eps, branch=name + )[2] self._n_input = len(masked_data) self._n_after_gal_mask = len(dict_tmp["flag"]) @@ -833,7 +1043,9 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): return m1, p1, m2, p2, ns @staticmethod - def get_variance_ivweights(data, sigma_eps, mask=None): + def get_variance_ivweights( + data, sigma_eps, mask=None, legacy=False, branch="NOSHEAR" + ): """Get Variance IVWEIGHTS. Compute variance and inverse-variance weights. @@ -858,8 +1070,11 @@ def get_variance_ivweights(data, sigma_eps, mask=None): weight """ - C11 = data["NGMIX_G1_ERR_NOSHEAR"] - C22 = data["NGMIX_G2_ERR_NOSHEAR"] + C11 = data[f"NGMIX_G1_ERR_{branch}"] + C22 = data[f"NGMIX_G2_ERR_{branch}"] + if not legacy: + C11 = C11**2 + C22 = C22**2 if mask is not None: C11 = C11[mask] C22 = C22[mask] @@ -883,11 +1098,173 @@ def _compute_calibration(self): raise ValueError(f"Invalid masking type '{self._masking_type}'") self._shear_response() - self._selection_response() - self._total_response() + if self._legacy_response: + self._selection_response() + self._total_response() + else: + self._prepare_weights() + result = self.response_estimates() + self.R_shear_global = result["shear"] + self.R_selection = result["selection"] + self.R = result["total"] + self.R_joint = result["joint"] + self.c_e, self.c_g = result["c_e"], result["c_g"] # self._shear_response_std(stat_operator=lambda x: # jackknif_weighted_average(x, np.ones_like(x))) + def _prepare_weights(self): + """Case d: fit W once, evaluate each branch's own SNR/size.""" + ma = self.mask_dict["ns"] + self.branches = { + b: (getattr(self, b)["g1"], getattr(self, b)["g2"]) + for b in response.BRANCHES + } + self.weight_metadata = dict( + scheme=self.weight_scheme, + step=self._step, + estimator="split", + W_policy="fixed", + des_min_count=self._des_min_count, + iv_policy="branch measurement variances", + c_units="c_e raw; c_g calibrated", + ) + if self.weight_scheme == "iv": + self.branch_weights = { + b: getattr(self, b)["w_iv"] for b in response.BRANCHES + } + else: + snr = self.ns["flux"] / self.ns["flux_err"] + size = self.ns["T"] / self.ns["Tpsf"] + if self._des_weight_product is None: + self.W = fit_w_des( + dict( + snr=snr[ma], + size_ratio=size[ma], + e1_uncal=self.ns["g1"][ma], + e2_uncal=self.ns["g2"][ma], + R_g11=self.R11, + R_g22=self.R22, + ), + self._des_num_bins, + self._snr_min, + self._snr_max, + self._rel_size_min, + self._rel_size_max, + min_count=self._des_min_count, + ) + ns_cell = ( + response.cell_index(snr, self.W[0]), + response.cell_index(size, self.W[1]), + ) + self.branch_weights = {} + self.branch_migration = {} + for b in response.BRANCHES: + d = getattr(self, b) + s, r = d["flux"] / d["flux_err"], d["T"] / d["Tpsf"] + self.branch_weights[b] = response.evaluate_w(self.W, s, r) + selected = self.mask_dict[b] + moved = (response.cell_index(s, self.W[0]) != ns_cell[0]) | ( + response.cell_index(r, self.W[1]) != ns_cell[1] + ) + self.branch_migration[b] = float(np.mean(moved[selected])) + self.ns["w_des"] = self.branch_weights["ns"] + self.ns["w"] = self.branch_weights["ns"] + self.branch_counts = {b: len(self.mask_dict[b]) for b in response.BRANCHES} + self.branch_weight_sums = { + b: self.branch_weights[b][self.mask_dict[b]].sum() + for b in response.BRANCHES + } + + def response_estimates(self, keep=None): + return response.responses( + self.branches, + self.mask_dict, + self.branch_weights, + self.R_shear, + self._step, + keep, + ) + + def refit_weights(self, keep): + """Refit DES W on retained S_ns, evaluate it on every parent branch. + + Edges and cuts stay fixed; cell moments and membership of the training + sample change. Artificial branches never train their own table. + """ + if self.weight_scheme != "des" or self._des_weight_product is not None: + return self.branch_weights + idx = self.mask_dict["ns"] + retained = keep[idx] + selected = idx[retained] + snr = self.ns["flux"] / self.ns["flux_err"] + size = self.ns["T"] / self.ns["Tpsf"] + table = response.fit_w( + snr[selected], + size[selected], + np.asarray(self.branches["ns"])[:, selected], + 0.5 * (self.R11[retained] + self.R22[retained]), + self.W[0], + self.W[1], + min_count=self._des_min_count, + ) + return { + b: response.evaluate_w( + table, + getattr(self, b)["flux"] / getattr(self, b)["flux_err"], + getattr(self, b)["T"] / getattr(self, b)["Tpsf"], + ) + for b in response.BRANCHES + } + + def response_jackknife(self, groups, refit_w=False, *, reference=False): + """Delete spatial groups, optionally refitting W from retained cell sums. + + reference=True keeps the original K full-parent rescans for validation. + Both paths use the full-sample edges and retained-count pooling rule. + """ + if refit_w and self.weight_scheme == "des" and self._des_weight_product is None: + if not reference: + from sp_validation.des_jackknife import accumulate + + return accumulate(self, groups).jackknife() + groups = np.asarray(groups) + if len(groups) != len(self.ns["g1"]): + raise ValueError("Spatial groups must align to the common parent rows") + labels = np.unique(groups) + if len(labels) < 2: + raise ValueError( + "Response jackknife requires at least two spatial groups" + ) + samples = [] + for label in labels: + keep = groups != label + weights = self.refit_weights(keep) + samples.append( + response.responses( + self.branches, + self.mask_dict, + weights, + self.R_shear, + self._step, + keep, + ) + ) + estimates = {key: np.array([s[key] for s in samples]) for key in samples[0]} + vectors = np.concatenate( + [x.reshape(len(labels), -1) for x in estimates.values()], axis=1 + ) + delta = vectors - vectors.mean(axis=0) + cov = (len(labels) - 1) / len(labels) * delta.T @ delta + return labels, estimates, cov + return response.jackknife( + self.branches, + self.mask_dict, + self.branch_weights, + self.R_shear, + self._step, + np.asarray(groups), + ) + def add_cuts(self, snr_min=10, snr_max=500, rel_size_min=0.5, rel_size_max=3.0): """Add Cuts. @@ -926,7 +1303,7 @@ def _masking_gal(self): [self.ns, self.m1, self.p1, self.m2, self.p2], ["ns", "m1", "p1", "m2", "p2"], ): - Tr_tmp = data["T"] + Tr_tmp = data["T"] if self._legacy_response else data["T"].copy() if self._size_corr_ell: Tr_tmp *= (1 - (data["g1"] ** 2 + data["g2"] ** 2)) / ( 1 + (data["g1"] ** 2 + data["g2"] ** 2) @@ -1099,7 +1476,6 @@ def _total_response(self): def mask_gal_size( T, Tpsf, rel_size_min, rel_size_max, size_corr_ell=False, g1=None, g2=None ): - Tr_tmp = T if size_corr_ell: Tr_tmp *= (1 - g1**2 + g2**2) / (1 + g1**2 + g2**2) @@ -1110,7 +1486,6 @@ def mask_gal_size( def mask_gal_SNR(SNR, snr_min, snr_max): - mask = (SNR > snr_min) & (SNR < snr_max) return mask diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index 69d1b36e..9b9c1378 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -414,7 +414,7 @@ def write_shape_catalog( signal-to-noise ratio, default is `None` g : np.ndarray, optional calibrated reduced shear estimate components, corrected for - multiplicative and additive bias, g = R^-1 g_uncal - c; + multiplicative and additive bias, g = R^-1 (g_uncal - c_e); expected type is arrays(2, ngal) of float; default is ``None`` (no calibrated shears written) g1_uncal, g2_uncal : np.ndarray, optional @@ -432,9 +432,9 @@ def write_shape_catalog( R_select : 2x2 matrix of float, optional Global selection response matrix, default is ``None`` c : array(2) of float, optional, default is ``None`` - additive shear bias + additive bias c_e in raw ellipticity units (c_g = R^-1 c_e) c_err : array(2) of float, optional, default is ``None`` - error of c + error of the raw weighted mean c_e; historical products may store scatter alpha_leakage : float, optional Mean scale-dependent PSF leakage, default is ``None`` sigma_epsilon: float, optional @@ -603,7 +603,7 @@ def write_shape_catalog( primary_header["c1_err"] = (c_err[0], "Standard deviation of c_1") primary_header["c2_err"] = (c_err[1], "Standard deviation of c_2") - primary_header["w"] = "DES weight" + primary_header["w"] = f"{w_type} weight" if sigma_epsilon is not None: primary_header["sig_eps"] = (sigma_epsilon, "Shape noise RMS") diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index 630cdc61..d81bdcc3 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -1085,14 +1085,18 @@ def compute_weights_gatti( purpose="weights", ) - cat_gal["w_des"] = calibration.get_w_des( - cat_gal, - num_bins, - snr_min=snr_min, - snr_max=snr_max, - size_ratio_min=size_ratio_min, - size_ratio_max=size_ratio_max, - ) + if gal_metacal.weight_scheme == "des": + cat_gal["w_des"] = gal_metacal.ns["w_des"][mask_metacal] + else: + cat_gal["w_des"] = calibration.get_w_des( + cat_gal, + num_bins, + snr_min=snr_min, + snr_max=snr_max, + size_ratio_min=size_ratio_min, + size_ratio_max=size_ratio_max, + ) + cat_gal["w_iv"] = gal_metacal.ns["w_iv"][mask_metacal] def compute_PSF_leakage( diff --git a/src/sp_validation/cosmo_val/catalog_characterization.py b/src/sp_validation/cosmo_val/catalog_characterization.py index e2df6d8e..cd5c34d8 100644 --- a/src/sp_validation/cosmo_val/catalog_characterization.py +++ b/src/sp_validation/cosmo_val/catalog_characterization.py @@ -407,10 +407,10 @@ def calculate_additive_bias(self): self._c2 = {} for ver in self.versions: self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] with self.results[ver].temporarily_read_data(): e1, e2, w = self._read_shear_cols(ver, "e1_col", "e2_col", "w_col") - self._c1[ver], self._c2[ver] = additive_bias(e1, e2, w, R) + R1, R2 = self._component_responses(ver) + self._c1[ver], self._c2[ver] = additive_bias(e1 / R1, e2 / R2, w, 1) self.print_done("Finished additive bias calculation.") @property diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 8159a1a0..20838075 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -668,7 +668,7 @@ def _calibrated_g(self, ver): """Calibrated shear components ``(g1, g2)`` for a catalog version. Applies additive-bias subtraction and the multiplicative response: - ``g = (e − c) / R``. For DES the response is the catalog-averaged + ``g = e / R − c_g``. Stored c is in calibrated shear units. For DES the response is the catalog-averaged per-component ``R11``/``R22`` (column names in the config); for every other version it is the scalar ``R`` from the config. @@ -676,12 +676,20 @@ def _calibrated_g(self, ver): context, since it reads ``dat_shear`` columns. """ e1, e2 = self._read_shear_cols(ver, "e1_col", "e2_col") + R1, R2 = self._component_responses(ver) + return e1 / R1 - self.c1[ver], e2 / R2 - self.c2[ver] + + def _component_responses(self, ver): + """Resolve responses identically for mean-c production and subtraction. + + Called within the catalogue's temporarily_read_data context. + """ if ver == "DES": R1 = np.average(self.results[ver].dat_shear[self.cc[ver]["shear"]["R11"]]) R2 = np.average(self.results[ver].dat_shear[self.cc[ver]["shear"]["R22"]]) else: R1 = R2 = self.cc[ver]["shear"]["R"] - return (e1 - self.c1[ver]) / R1, (e2 - self.c2[ver]) / R2 + return R1, R2 @property def colors(self): diff --git a/src/sp_validation/des_jackknife.py b/src/sp_validation/des_jackknife.py new file mode 100644 index 00000000..6dca990e --- /dev/null +++ b/src/sp_validation/des_jackknife.py @@ -0,0 +1,189 @@ +"""Exact refit-DES deletions from fixed-cell sufficient statistics. + +Storage is O(groups * cells), independent of parent catalogue length. Feed +selected branch chunks with add_branch; no parent-sized scratch arrays are kept. +""" + +import numpy as np + +from sp_validation import response + + +def pooled_weight(moments, shape, min_count): + """Reapply the concentric-square rule to retained count, sum R and sum e². + + Integral images make each window query O(1). All sparse cells are evaluated + together at each radius; with the fixed 20x20 production grid this is O(cells). + Original counts, not pooled counts, determine which cells need pooling. + """ + count, rsum, esum = moments + weight = np.divide( + rsum**2, count * esum, out=np.zeros_like(rsum), where=count * esum > 0 + ) + if min_count < 0: + raise ValueError("min_count must be nonnegative") + pending = np.flatnonzero(count < min_count) + if not len(pending): + return weight + # Sparse windows subtract large prefix sums. Extended precision avoids + # cancellation at the high-index edges (visible even on the small grids). + image = np.pad( + moments.astype(np.longdouble).reshape(3, *shape), + ((0, 0), (1, 0), (1, 0)), + ) + image = image.cumsum(axis=1).cumsum(axis=2) + for radius in range(1, max(shape)): + i, j = np.unravel_index(pending, shape) + lo_i, hi_i = np.maximum(i - radius, 0), np.minimum(i + radius + 1, shape[0]) + lo_j, hi_j = np.maximum(j - radius, 0), np.minimum(j + radius + 1, shape[1]) + sums = ( + image[:, hi_i, hi_j] + - image[:, lo_i, hi_j] + - image[:, hi_i, lo_j] + + image[:, lo_i, lo_j] + ) + ready = (sums[0] >= min_count) | (radius == max(shape) - 1) + n, r, e2 = sums[:, ready] + weight[pending[ready]] = np.divide( + r**2, n * e2, out=np.zeros_like(r), where=n * e2 > 0 + ) + pending = pending[~ready] + if not len(pending): + break + return weight + + +class DESJackknife: + """Accumulate raw moments, then apply each deletion's refitted cell weights. + + The caller supplies the full-sample edges, never deletion-dependent edges. + Per branch/cell we need count, sum of no-shear shapes (selection response), + and sum of that branch's shapes (joint response). The ns branch also stores + sum R_g and the three valid-training moments used by fit_w. + """ + + def __init__(self, labels, edges_s, edges_r, step, min_count=3): + self.labels = np.asarray(labels) + if len(labels) < 2: + raise ValueError("Response jackknife requires at least two spatial groups") + self.edges = edges_s, edges_r + self.shape = len(edges_s) - 1, len(edges_r) - 1 + self.cells = np.prod(self.shape) + self.step, self.min_count = step, min_count + shape = len(labels), self.cells + self.sums = {b: np.zeros((5, *shape)) for b in response.BRANCHES} + self.rg = np.zeros((4, *shape)) + self.training = np.zeros((3, *shape)) + + def add_branch(self, b, groups, snr, size, ns_e, e, rg=None): + """Add selected rows of one branch; rg aligns to selected ns rows only.""" + finite = np.isfinite(snr) & np.isfinite(size) + cell = response.cell_index(snr, self.edges[0]) * self.shape[ + 1 + ] + response.cell_index(size, self.edges[1]) + index = np.searchsorted(self.labels, groups) * self.cells + cell + + def add(target, values, valid=finite): + ix = index[valid] + if not len(ix): + return + v = None if values is None else values[valid] + lo, hi = ix.min(), ix.max() + 1 + flat = target.reshape(-1) + if hi - lo <= 2 * len(ix): + # Sorted spatial catalogues usually touch only a few groups + # per chunk. Never clear a groups*cells scratch per chunk: + # cap the dense span at O(chunk rows), otherwise scatter. + flat[lo:hi] += np.bincount(ix - lo, weights=v, minlength=hi - lo) + else: + np.add.at(flat, ix, 1 if v is None else v) + + add(self.sums[b][0], None) + for i in range(2): + add(self.sums[b][1 + i], ns_e[i]) + add(self.sums[b][3 + i], e[i]) + if b == "ns": + for i, values in enumerate(rg.reshape(4, -1)): + add(self.rg[i], values) + r = 0.5 * (rg[0, 0] + rg[1, 1]) + valid = finite & np.isfinite(e).all(axis=0) & np.isfinite(r) + add(self.training[0], None, valid) + add(self.training[1], r, valid) + add(self.training[2], np.mean(e**2, axis=0), valid) + + def deleted_weight(self, group): + """Return the retained table, useful for reference comparisons.""" + moments = self.training.sum(axis=1) - self.training[:, group] + return pooled_weight(moments, self.shape, self.min_count).reshape(self.shape) + + def jackknife(self): + """Return the same response/c estimates and covariance as the row path.""" + totals = {b: x.sum(axis=1) for b, x in self.sums.items()} + training_total = self.training.sum(axis=1) + rg_total = self.rg.sum(axis=1) + samples = [] + for k in range(len(self.labels)): + w = pooled_weight( + training_total - self.training[:, k], self.shape, self.min_count + ) + means = {} + den = {} + for b in response.BRANCHES: + weighted = (totals[b] - self.sums[b][:, k]) @ w + den[b] = weighted[0] + if den[b] <= 0: + raise ValueError( + f"Empty weighted population after spatial deletion: {b}" + ) + means[b] = weighted[1:] / den[b] + shear = ((rg_total - self.rg[:, k]) @ w / den["ns"]).reshape(2, 2) + selection, joint = np.empty((2, 2)), np.empty((2, 2)) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + selection[:, j] = (means[p][:2] - means[m][:2]) / (2 * self.step) + joint[:, j] = (means[p][2:] - means[m][2:]) / (2 * self.step) + total = shear + selection + ce = means["ns"][:2] + samples.append( + dict( + shear=shear, + selection=selection, + total=total, + joint=joint, + c_e=ce, + c_g=np.linalg.solve(total, ce), + ) + ) + estimates = {key: np.array([s[key] for s in samples]) for key in samples[0]} + vectors = np.concatenate( + [x.reshape(len(self.labels), -1) for x in estimates.values()], axis=1 + ) + delta = vectors - vectors.mean(axis=0) + cov = (len(self.labels) - 1) / len(self.labels) * delta.T @ delta + return self.labels, estimates, cov + + +def accumulate(gm, groups, chunk_size=500_000): + """Read an existing metacal parent in bounded chunks, without copying it.""" + groups = np.asarray(groups) + if len(groups) != len(gm.ns["g1"]): + raise ValueError("Spatial groups must align to the common parent rows") + # np.unique(return_inverse=True) on 350M rows would itself require GBs. + labels = np.empty(0, dtype=groups.dtype) + for start in range(0, len(groups), chunk_size): + labels = np.union1d(labels, np.unique(groups[start : start + chunk_size])) + acc = DESJackknife(labels, gm.W[0], gm.W[1], gm._step, gm._des_min_count) + for b in response.BRANCHES: + selected = gm.mask_dict[b] + d = getattr(gm, b) + for start in range(0, len(selected), chunk_size): + idx = selected[start : start + chunk_size] + acc.add_branch( + b, + groups[idx], + d["flux"][idx] / d["flux_err"][idx], + d["T"][idx] / d["Tpsf"][idx], + np.array([x[idx] for x in gm.branches["ns"]]), + np.array([x[idx] for x in gm.branches[b]]), + gm.R_shear[:, :, start : start + chunk_size] if b == "ns" else None, + ) + return acc diff --git a/src/sp_validation/image_sim_config.py b/src/sp_validation/image_sim_config.py new file mode 100644 index 00000000..a0a368cc --- /dev/null +++ b/src/sp_validation/image_sim_config.py @@ -0,0 +1,33 @@ +"""Image-simulation calibration policy (no catalogue I/O).""" + +from copy import deepcopy + + +def calibration_config(config, w_cols, source="external", product=None): + """Use a named data calibration table by default for DES image sims. + + Explicit ``source='self'`` retains sample-fitted W as a diagnostic. + Non-DES runs retain their existing calibration configuration. + """ + config = deepcopy(config) + if "w_des" not in w_cols: + return config + if source not in ("external", "self"): + raise ValueError("des_weight_source must be external or self") + if source == "external" and ( + not isinstance(product, dict) + or not product.get("name") + or not product.get("path") + ): + raise ValueError( + "DES image sims require des_weight_product with name and path; " + "des_weight_source: self is diagnostic only" + ) + cm = config["metacal"] + cm.pop("global_R_weight", None) + cm.update(weight_scheme="des", w_col="w_des", additive_correction=False) + if source == "external": + cm["des_weight_product"] = product.copy() + else: + cm.pop("des_weight_product", None) + return config diff --git a/src/sp_validation/image_sims.py b/src/sp_validation/image_sims.py index e259109d..54371f21 100644 --- a/src/sp_validation/image_sims.py +++ b/src/sp_validation/image_sims.py @@ -51,8 +51,14 @@ def _load_cat(path, w_cols): scheme in a multi-weight run. """ with fits.open(path) as hdul: + if hdul[0].header.get("RAPPLIED", False): + from sp_validation.response import validate_weight_scheme + + for col in w_cols: + validate_weight_scheme(hdul[0].header["WGHTSCH"], col) data = hdul[1].data cat = { + "weight_scheme": hdul[0].header.get("WGHTSCH"), "ra": data["RA"].copy(), "dec": data["Dec"].copy(), "e1": data["e1"].copy(), @@ -65,6 +71,27 @@ def _load_cat(path, w_cols): if _is_unweighted(scheme) else data[scheme].copy() ) + if cat["weight_scheme"] is not None: + if hdul[0].header.get("MEANSUB", False): + raise ValueError( + "Constant-shear simulations must not subtract branch means" + ) + cat["raw"] = np.array([data["e1_uncal"], data["e2_uncal"]]) + cat["tiles"] = np.char.strip(np.asarray(data["TILE_ID"]).astype(str)) + if "R_JK" not in hdul: + raise ValueError( + "Modern m uncertainties require shared-tile R_JK responses" + ) + jk = hdul["R_JK"].data + cat["jk_labels"] = np.char.strip(np.asarray(jk["TILE_ID"]).astype(str)) + cat["jk_R"] = np.array( + [[jk[f"R_{i}{j}"] for j in (1, 2)] for i in (1, 2)] + ).transpose(2, 0, 1) + cat["W_policy"] = hdul[0].header.get("WPOLICY", "unknown") + if cat["W_policy"] == "refit": + # These means include each deletion's refitted numerator weights, + # not merely a refitted response applied to fixed-W means. + cat["jk_g"] = np.array([jk["c_g1"], jk["c_g2"]]).T return cat @@ -97,13 +124,14 @@ class ImageSimMBias: (required); ``"none"`` (or a ``null`` entry) means unit weights, any other entry is a FITS column name. The **first** entry is the primary result surfaced at the top level of ``run()``'s output. Our - fiducial run leads with the unweighted scheme (``["none", ...]``), - per the #227 verdict that shape weights are excluded from sim - calibration; the unweighted m also avoids the ``cov(w, e)`` residual - weighted estimators carry on constant-shear sims. + modern production run uses the weight matching the stored response + (DES case d: ``["w_des"]``) and ``pair_match=False``. + Symmetric pair weights with a full-population response are refused. - w_col : str or None, *deprecated* single weight scheme; accepted for back-compat and used as ``[w_col]`` only when ``w_cols`` is absent. - - n_bootstrap : int, number of bootstrap resamples for errors (required) + - conditional_bootstrap : bool, optional fixed-response object-bootstrap + diagnostic for modern products (default False); never the headline error + - n_bootstrap : int, number of conditional diagnostic draws (required) - bootstrap_seed : int, seed for the per-pair bootstrap RNG (required); makes the bootstrap errors bit-reproducible. The resample indices are drawn once per pair and shared across every weight scheme, so the @@ -160,6 +188,12 @@ def load_catalogs(self, verbose=True): if verbose: print(f" Loading {path}") self.cats[name] = _load_cat(path, self.w_cols) + if self.pair_match and self.cats[name]["weight_scheme"] is not None: + raise ValueError( + "Modern response products require pair_match=False: headline m " + "uses full populations and separately normalized branch weights; " + "symmetric pair weights do not match the stored response." + ) if verbose: print(f" {len(self.cats[name]['ra'])} objects") @@ -228,24 +262,13 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): e_p = self.cats[name_p][e_key][idx_p] e_m = self.cats[name_m][e_key][idx_m] - # Draw the bootstrap resample indices *once*, before the weight-scheme - # loop, and reuse them for every scheme -- the schemes then differ only - # in their weighting, never in their draws (so a scheme comparison is a - # clean weighting comparison). Pre-drawing the full ``(n_boot, n)`` block - # in one call is bit-identical to drawing ``rng.integers(0, n, n)`` once - # per replicate (numpy fills the block row-major), so the numbers match a - # single-scheme, per-iteration bootstrap to the last bit. - rng = np.random.default_rng(seed=self.boot_seed) - if self.pair_match: - n = len(e_p) - ib = rng.integers(0, n, (self.n_boot, n)) - else: - n_p, n_m = len(e_p), len(e_m) - ib_p = rng.integers(0, n_p, (self.n_boot, n_p)) - ib_m = rng.integers(0, n_m, (self.n_boot, n_m)) + # Generate one draw at a time: no catalogue-sized n_boot x N array. + # Reset the seed per scheme so comparisons use identical draws. + n_p, n_m = len(e_p), len(e_m) res = {} for scheme in self.w_cols: + rng = np.random.default_rng(seed=self.boot_seed) w_p = self.cats[name_p]["w"][scheme][idx_p] w_m = self.cats[name_m]["w"][scheme][idx_m] m_boot = np.empty(self.n_boot) @@ -265,8 +288,9 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): # sims, so the per-object cancellation in `d` is preserved in # the error estimate. for i in range(self.n_boot): - m_boot[i] = np.average(d[ib[i]], weights=w[ib[i]]) - c_boot[i] = np.average(s[ib[i]], weights=w[ib[i]]) + ib = rng.integers(0, n_p, n_p) + m_boot[i] = np.average(d[ib], weights=w[ib]) + c_boot[i] = np.average(s[ib], weights=w[ib]) else: # No matching: difference/sum the two independent weighted means. mean_ep = np.average(e_p, weights=w_p) @@ -278,8 +302,10 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): # Unpaired bootstrap: the +g and -g arrays generally differ in # length, so each side is resampled independently per replicate. for i in range(self.n_boot): - ep_b = np.average(e_p[ib_p[i]], weights=w_p[ib_p[i]]) - em_b = np.average(e_m[ib_m[i]], weights=w_m[ib_m[i]]) + ib_p = rng.integers(0, n_p, n_p) + ib_m = rng.integers(0, n_m, n_m) + ep_b = np.average(e_p[ib_p], weights=w_p[ib_p]) + em_b = np.average(e_m[ib_m], weights=w_m[ib_m]) m_boot[i] = (ep_b - em_b) / (2 * self.g_in) - 1 c_boot[i] = (ep_b + em_b) / 2 @@ -287,9 +313,109 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): return res + def _run_tile_jackknife(self, verbose): + """Delete the same parent tile on every external-shear branch. + + R_JK carries responses recomputed upstream from all five metacal + selections, including sheared-only selected rows. Raw selected shapes + supply each deletion's numerator; never invert the full-sample response + to approximate a deleted response. + """ + cats = [self.cats[name] for p, m, _ in self.pairs for name in (p, m)] + labels = np.sort(cats[0]["jk_labels"]) + if len(labels) < 2: + raise ValueError("At least two shared tiles are required") + for cat in cats: + if not np.array_equal(np.sort(cat["jk_labels"]), labels): + raise ValueError( + "External branches must have identical parent R_JK tiles" + ) + if not np.all(np.isin(cat["tiles"], labels)): + raise ValueError("Selected tiles are missing from parent R_JK") + order = [ + key for _, _, comp in self.pairs for key in (f"m{comp + 1}", f"c{comp + 1}") + ] + results = dict( + weights={}, + estimator="full_population_branch_normalized", + uncertainty="shared_tile_response_jackknife", + W_policy=cats[0]["W_policy"], + covariance_order=order, + jackknife_tiles=labels.tolist(), + ) + for scheme in self.w_cols: + estimates, deleted = [], [] + for p, m, comp in self.pairs: + means, jk_means = [], [] + for name in (p, m): + cat = self.cats[name] + w = cat["w"][scheme] + means.append(np.average(cat[f"e{comp + 1}"], weights=w)) + if cat["W_policy"] == "refit": + by_label = dict(zip(cat["jk_labels"], cat["jk_g"], strict=True)) + jk_means.append(np.array([by_label[t][comp] for t in labels])) + continue + by_label = dict(zip(cat["jk_labels"], cat["jk_R"], strict=True)) + calibrated = [] + for tile in labels: + keep = cat["tiles"] != tile + raw_mean = np.average( + cat["raw"][:, keep], axis=1, weights=w[keep] + ) + calibrated.append( + np.linalg.solve(by_label[tile], raw_mean)[comp] + ) + jk_means.append(np.asarray(calibrated)) + estimates.extend( + ( + (means[0] - means[1]) / (2 * self.g_in) - 1, + (means[0] + means[1]) / 2, + ) + ) + deleted.extend( + ( + (jk_means[0] - jk_means[1]) / (2 * self.g_in) - 1, + (jk_means[0] + jk_means[1]) / 2, + ) + ) + draws = np.array(deleted).T + delta = draws - draws.mean(axis=0) + cov = (len(labels) - 1) / len(labels) * delta.T @ delta + result = dict(zip(order, map(float, estimates), strict=True)) + result.update( + { + f"{key}_err": float(error) + for key, error in zip(order, np.sqrt(np.diag(cov)), strict=True) + } + ) + result["mc_covariance"] = cov.tolist() + result["mc_jackknife"] = draws.tolist() + results["weights"][scheme] = result + if verbose: + print( + f"[{scheme}] shared-tile response jackknife ({len(labels)} tiles)" + ) + for key in order: + print(f" {key} = {result[key]:+.6f} ± {result[key + '_err']:.6f}") + results.update(results["weights"][self.w_cols[0]]) + # Optional, explicitly conditional diagnostic; never replace science errors. + if self.cfg.get("conditional_bootstrap", False): + diagnostic = {scheme: {} for scheme in self.w_cols} + for p, m, comp in self.pairs: + for scheme, values in self._m_c_pair(p, m, comp, False).items(): + diagnostic[scheme][f"m{comp + 1}_err"] = float(values[1]) + diagnostic[scheme][f"c{comp + 1}_err"] = float(values[3]) + results["conditional_object_bootstrap_fixed_response"] = diagnostic + return results + def run(self, verbose=True): """Compute m and c for both shear components and every weight scheme. + Modern products use shared-tile, response-recalibrating jackknife errors + and export mc_covariance in covariance_order, including m/c cross terms. + W_policy records whether the upstream deleted responses held W fixed. + Legacy object-bootstrap errors are labelled conditional on fixed R. + Returns ------- dict @@ -299,7 +425,20 @@ def run(self, verbose=True): reader that wants the headline m/c never has to know the scheme name. """ - results = {"weights": {scheme: {} for scheme in self.w_cols}} + modern = [cat.get("weight_scheme") is not None for cat in self.cats.values()] + if any(modern): + if not all(modern) or self.pair_match: + raise ValueError( + "Modern response products require full populations on all branches" + ) + return self._run_tile_jackknife(verbose) + results = { + "weights": {scheme: {} for scheme in self.w_cols}, + "estimator": "legacy_symmetric_pairs" + if self.pair_match + else "full_population", + "uncertainty": "conditional_object_bootstrap_fixed_response", + } for name_p, name_m, comp in self.pairs: label = f"g{comp + 1}" if verbose: diff --git a/src/sp_validation/io.py b/src/sp_validation/io.py index b95b90df..ac12238a 100644 --- a/src/sp_validation/io.py +++ b/src/sp_validation/io.py @@ -365,12 +365,29 @@ def open_catalogue(path, hdu=None, column_map=None): return catalogue.table() -def open_entry(entry): - """Return the catalogue a config block names (e.g. a cat_config ``shear``). +def validate_calibrated_entry(entry): + """Validate response/weight identity, also for file-based reader adapters.""" + if "w_col" in entry and not h5py.is_hdf5(entry["path"]): + with fits.open(entry["path"], memmap=True) as hdus: + header = hdus[0].header + if header.get("RAPPLIED", False): + from sp_validation.response import validate_weight_scheme + + validate_weight_scheme(header["WGHTSCH"], entry["w_col"]) + if ( + entry.get("R", 1) != 1 + or entry.get("R11") is not None + or entry.get("R22") is not None + ): + raise ValueError( + "Response already applied: cat_config R must be 1 " + "with no per-object response columns" + ) - The block's ``path``, optional ``hdu`` and optional ``column_map`` go to - ``open_catalogue``. - """ + +def open_entry(entry): + """Read a config entry, checking calibrated-response/weight metadata.""" + validate_calibrated_entry(entry) return open_catalogue( entry["path"], hdu=entry.get("hdu"), column_map=entry.get("column_map") ) diff --git a/src/sp_validation/response.py b/src/sp_validation/response.py new file mode 100644 index 00000000..7d96bb3d --- /dev/null +++ b/src/sp_validation/response.py @@ -0,0 +1,188 @@ +"""Weighted metacal response primitives. + +Definitions: Martin Kilbinger's 6 October 2026 slide, p14 (S1), and +/n23data1/mkilbing/imsim_wdes_response/wdes_response.py, case d. +Spatial deletion follows his response_jackknife, with fixed-W +comparisons and retained-sample refits in calibration.metacal. +No detection response is inferred from aligned metacal branches. +""" + +import numpy as np + +BRANCHES = ("ns", "p1", "m1", "p2", "m2") + + +def branch_error_columns(data): + """Carry available sheared G_ERRs through extraction without requiring them. + + Historical merged products omitted them. DES can still use those products; + modern IV independently requires a complete set and never freezes weights. + """ + return { + key: data[key] + for shear in ("1P", "1M", "2P", "2M") + for i in (1, 2) + if (key := f"NGMIX_G{i}_ERR_{shear}") in data.dtype.names + } + + +def cell_index(x, edges): + """Right-closed cells; out-of-range finite values use nearest edge cell.""" + return np.clip(np.searchsorted(edges, x, side="left") - 1, 0, len(edges) - 2) + + +def evaluate_w(table, snr, size): + """Evaluate one fixed table, including rows absent from the training sample. + + Unpooled empty/invalid cells and nonfinite coordinates get zero weight. + """ + es, er, values, _ = table + w = values[cell_index(snr, es), cell_index(size, er)].copy() + w[~np.isfinite(snr) | ~np.isfinite(size) | ~np.isfinite(w)] = 0 + return w + + +def fit_w(snr, size, e, response, edges_s, edges_r, min_count=0): + """Fit W = mean(trace R/2)^2 / mean(|e|^2/2), unweighted per cell. + + Uses second moments, not centered variances. Fitting and evaluation share + the same boundary convention. Below min_count, pool moments over the + smallest clipped concentric square of cells reaching min_count; if the + entire training set is smaller, use its moments. Populated cells are exact. + The returned counts are unmerged training counts. + """ + shape = (len(edges_s) - 1, len(edges_r) - 1) + idx = cell_index(snr, edges_s) * shape[1] + cell_index(size, edges_r) + valid = np.isfinite(snr) & np.isfinite(size) + valid &= np.all(np.isfinite(e), axis=0) & np.isfinite(response) + idx = idx[valid] + count = np.bincount(idx, minlength=np.prod(shape)) + rsum = np.bincount(idx, weights=response[valid], minlength=len(count)) + esum = np.bincount( + idx, weights=np.mean(e[:, valid] ** 2, axis=0), minlength=len(count) + ) + w = np.divide( + rsum**2, count * esum, out=np.zeros_like(rsum), where=(count * esum) > 0 + ) + if min_count < 0: + raise ValueError("min_count must be nonnegative") + # Smallest concentric square in cell-index space (clipped at the edges). + # Pool sufficient statistics, never weights; retain the original counts. + if min_count: + counts, rsums, esums = (x.reshape(shape) for x in (count, rsum, esum)) + for i, j in np.argwhere(counts < min_count): + for radius in range(1, max(shape)): + window = ( + slice(max(0, i - radius), min(shape[0], i + radius + 1)), + slice(max(0, j - radius), min(shape[1], j + radius + 1)), + ) + n = counts[window].sum() + if n >= min_count or radius == max(shape) - 1: + r, e2 = rsums[window].sum(), esums[window].sum() + w[i * shape[1] + j] = r**2 / (n * e2) if n * e2 > 0 else 0 + break + return edges_s, edges_r, w.reshape(shape), count.reshape(shape) + + +def responses(branches, masks, weights, rg, step, keep=None): + """S1 split and joint derivatives, each mean separately normalized. + + rg is the per-object response on no-shear-selected rows, aligned to + masks['ns']. All other arrays are aligned to the common parent rows. + """ + selected = {b: idx if keep is None else idx[keep[idx]] for b, idx in masks.items()} + + def mean(data, b): + idx = selected[b] + w = weights[b][idx] + if not len(idx) or not np.isfinite(w).all() or w.sum() <= 0: + raise ValueError(f"Empty or invalid weighted population: {b}") + return np.array([np.average(x[idx], weights=w) for x in data]) + + ns = masks["ns"] + k = np.ones(len(ns), bool) if keep is None else keep[ns] + shear = np.average(rg[:, :, k], axis=2, weights=weights["ns"][ns][k]) + selection = np.empty((2, 2)) + joint = np.empty((2, 2)) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + selection[:, j] = (mean(branches["ns"], p) - mean(branches["ns"], m)) / ( + 2 * step + ) + joint[:, j] = (mean(branches[p], p) - mean(branches[m], m)) / (2 * step) + total = shear + selection + ce = mean(branches["ns"], "ns") + cg = np.linalg.solve(total, ce) + return dict( + shear=shear, selection=selection, total=total, joint=joint, c_e=ce, c_g=cg + ) + + +def jackknife(branches, masks, weights, rg, step, groups): + """Delete common spatial groups in all branches; conditional on fixed W. + + Covariance order: shear, selection, total, joint (row-major), c_e, c_g. + Extends Martin's response_jackknife to additive bias and full covariance. + """ + if len(groups) != len(branches["ns"][0]): + raise ValueError("Spatial groups must align to the common parent rows") + labels, group_index = np.unique(groups, return_inverse=True) + if len(labels) < 2: + raise ValueError("Response jackknife requires at least two spatial groups") + + # Fixed-W sufficient sums: O(N) rather than rereading N rows K times. + def deleted_sum(values, index): + sums = np.bincount(index, weights=values, minlength=len(labels)) + return sums.sum() - sums + + den = {} + for b in masks: + idx = masks[b] + den[b] = deleted_sum(weights[b][idx], group_index[idx]) + if np.any(den[b] <= 0): + raise ValueError(f"Empty weighted population after spatial deletion: {b}") + + def means(data, b): + idx = masks[b] + return np.array( + [ + deleted_sum(x[idx] * weights[b][idx], group_index[idx]) / den[b] + for x in data + ] + ).T + + ns = masks["ns"] + shear = np.empty((len(labels), 2, 2)) + selection = np.empty_like(shear) + joint = np.empty_like(shear) + for i in range(2): + for j in range(2): + shear[:, i, j] = ( + deleted_sum(rg[i, j] * weights["ns"][ns], group_index[ns]) / den["ns"] + ) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + selection[:, :, j] = (means(branches["ns"], p) - means(branches["ns"], m)) / ( + 2 * step + ) + joint[:, :, j] = (means(branches[p], p) - means(branches[m], m)) / (2 * step) + total = shear + selection + ce = means(branches["ns"], "ns") + cg = np.linalg.solve(total, ce[..., None])[..., 0] + estimates = dict( + shear=shear, selection=selection, total=total, joint=joint, c_e=ce, c_g=cg + ) + vectors = np.concatenate( + [estimates[k].reshape(len(labels), -1) for k in estimates], axis=1 + ) + delta = vectors - vectors.mean(axis=0) + cov = (len(labels) - 1) / len(labels) * delta.T @ delta + return labels, estimates, cov + + +def validate_weight_scheme(scheme, w_col): + """Reject statistic/calibration mismatches, rather than relabel a response.""" + if scheme not in ("des", "iv"): + raise ValueError("weight_scheme must be des or iv") + if w_col != f"w_{scheme}": + raise ValueError( + f"Calibration weight_scheme={scheme} disagrees with w_col={w_col}" + ) diff --git a/src/sp_validation/response_diagnostics.py b/src/sp_validation/response_diagnostics.py new file mode 100644 index 00000000..9ed93d31 --- /dev/null +++ b/src/sp_validation/response_diagnostics.py @@ -0,0 +1,189 @@ +"""Response-only diagnostics, not cosmological signal measurements. + +Pair products use raw per-object R_g, not total selection/weight response. +They cannot certify adequacy of a global total response on their own (#23). +""" + +import numpy as np + + +def pair_products(ra, dec, weight, rg, min_sep=1, max_sep=120, nbins=8): + """Symmetrized angle-binned _ww via TreeCorr. + + Coordinates are degrees, separations arcminutes. Nonzero min_sep excludes + self-pairs; cross-scalar correlations include both pair orientations. + Returns all 16 tensor products, not just the scalar trace approximation. + """ + import treecorr + + scalar = rg.reshape(4, -1) + cats = [ + treecorr.Catalog(ra=ra, dec=dec, ra_units="deg", dec_units="deg", w=weight, k=x) + for x in scalar + ] + products = np.empty((nbins, 4, 4)) + for i in range(4): + for j in range(i, 4): + corr = treecorr.KKCorrelation( + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + sep_units="arcmin", + bin_slop=0, + ) + corr.process(cats[i], None if i == j else cats[j], num_threads=4) + products[:, i, j] = products[:, j, i] = np.where( + corr.weight > 0, corr.xi, np.nan + ) + means = np.average(scalar, axis=1, weights=weight) + return dict( + theta=np.exp(corr.meanlogr), + products=products, + global_products=np.outer(means, means), + pair_weight=corr.weight, + ) + + +def scalar_pair_response( + ra, + dec, + weight, + r, + *, + patch=None, + patch_centers=None, + min_sep=1, + max_sep=250, + nbins=20, + bin_slop=0.1, +): + """Stage-1 _ww / _w**2 - 1, r = trace(R_g)/2. + + Delete the identical patch from pair and global statistics, then form each + ratio. This includes covariance of the pair and global response, unlike + an error bar on the pair product alone. Science selection and weights are + fixed inputs; selection/weight derivatives and anisotropic mixing aren't + measured here. No ellipticity correlation is computed. + """ + import treecorr + + cat = treecorr.Catalog( + ra=ra, + dec=dec, + ra_units="deg", + dec_units="deg", + w=weight, + k=r, + patch=patch, + patch_centers=patch_centers, + ) + corr = treecorr.KKCorrelation( + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + sep_units="arcmin", + bin_slop=bin_slop, + var_method="jackknife", + cross_patch_weight="simple", + ) + corr.process(cat, num_threads=4) + pairs, _ = treecorr.build_multi_cov_design_matrix( + [corr], "jackknife", cross_patch_weight="simple" + ) + group = cat.patch + den = np.bincount(group, weights=weight, minlength=cat.npatch) + num = np.bincount(group, weights=weight * r, minlength=cat.npatch) + global_deleted = (num.sum() - num) / (den.sum() - den) + global_r = num.sum() / den.sum() + deleted = pairs / global_deleted[:, None] ** 2 - 1 + delta = deleted - deleted.mean(axis=0) + covariance = (len(deleted) - 1) / len(deleted) * delta.T @ delta + return dict( + theta=np.exp(corr.meanlogr), + pair=corr.xi, + global_r=global_r, + fraction=corr.xi / global_r**2 - 1, + deleted=deleted, + global_deleted=global_deleted, + covariance=covariance, + error=np.sqrt(np.diag(covariance)), + pair_weight=corr.weight, + ) + + +def plot_response(gm, parent_groups, ra, dec, output): + """Weighted/unweighted R_g, spatial split/joint samples, and patch map.""" + import matplotlib.pyplot as plt + + selected = gm.mask_dict["ns"] + w = gm.branch_weights["ns"][selected] + labels, jk, _ = gm.response_jackknife(parent_groups) + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + ax = axes[i, j] + x = gm.R_shear[i, j] + limits = np.percentile(x, [0.5, 99.5]) + ax.hist( + x, + bins=60, + range=limits, + density=True, + histtype="step", + label="Unweighted", + ) + ax.hist( + x, + bins=60, + range=limits, + weights=w, + density=True, + histtype="step", + label="DES weighted", + ) + ax.set(xlabel=f"Per-object Rg,{i + 1}{j + 1}", ylabel="Density") + ax.legend() + fig.tight_layout() + fig.savefig(output / "response_distributions.png", dpi=160) + plt.close(fig) + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + ax = axes[i, j] + for key in ("selection", "total", "joint"): + ax.plot(np.arange(len(labels)), jk[key][:, i, j], ".", label=key) + ax.set(xlabel="Deleted spatial group", ylabel=f"Response {i + 1}{j + 1}") + ax.legend() + fig.tight_layout() + fig.savefig(output / "response_jackknife.png", dpi=160) + plt.close(fig) + groups = parent_groups[selected] + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + values, x, y = [], [], [] + for label in labels: + keep = groups == label + if not keep.any(): + continue + x.append(np.mean(ra[keep])) + y.append(np.mean(dec[keep])) + values.append(np.average(gm.R_shear[i, j, keep], weights=w[keep])) + im = axes[i, j].scatter(x, y, c=values, s=45) + fig.colorbar(im, ax=axes[i, j], label=f"Mean Rg,{i + 1}{j + 1}") + axes[i, j].set(xlabel="RA [deg]", ylabel="Dec [deg]") + fig.tight_layout() + fig.savefig(output / "response_map.png", dpi=160) + plt.close(fig) + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + delta = jk["total"][:, i, j] - jk["joint"][:, i, j] + axes[i, j].plot(np.arange(len(labels)), delta, ".") + axes[i, j].axhline(gm.R[i, j] - gm.R_joint[i, j], color="k", lw=0.8) + axes[i, j].set( + xlabel="Deleted spatial group", ylabel=f"Split − joint, {i + 1}{j + 1}" + ) + fig.tight_layout() + fig.savefig(output / "split_joint.png", dpi=160) + plt.close(fig) diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index e744fdf8..d18a505c 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -96,6 +96,7 @@ def _columns(self, block): def __call__(self, block): if block not in self._cache: entry = self._info[block] + io.validate_calibrated_entry(entry) with io.Catalogue( entry["path"], hdu=entry.get("hdu"), column_map=entry.get("column_map") ) as catalogue: diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index f5f3e3c0..9a03b436 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -1,10 +1,11 @@ # Image-simulation m-bias workflow configuration. # # OPERATIONAL keys default here and nowhere else -- the .smk reads them bare, so -# this file is their single home. SCIENCE keys have no default anywhere and -# appear below only as commented template lines: they fix the estimator's -# behaviour, so each run must state them in a run config layered on top. A run -# config that omits one fails at DAG parse, naming the key; an unknown key under +# this file is their single home. Run-specific SCIENCE keys appear below as +# commented template lines and must be stated in a layered run config. +# DES has one shared production policy: use an independent named data table; +# sample-fitted tables require the explicit diagnostic override below. +# A run missing a required science key fails at DAG parse; an unknown key under # `image_sims:` fails as a typo. image_sims: @@ -58,6 +59,18 @@ image_sims: extract_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/extract_info.py calibrate_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/calibrate_comprehensive_cat.py + # DES sims must apply the same fixed table as the data they calibrate. + # Supply a named modern calibration FITS product (W_DES/W_EDGES/W_COUNT, + # with coordinate metadata), not a released catalogue with only w_des. + # No public v1.4.6.3 table with verified compatible edges is available. + # This intentionally blocks a DES production run until its data product is set. + des_weight_source: external + des_weight_product: null + # des_weight_product: + # name: + # path: /absolute/path/to/shape_catalog_cut_ngmix.fits + # des_weight_source: self # explicitly sample-fitted diagnostic, not production + # --- science knobs (REQUIRED in the run config; no default) ----------- # Copy these into your run config and set them. There is deliberately no # default: each fixes the estimator's scientific behaviour, so a run must @@ -66,11 +79,8 @@ image_sims: # # mask_config: config/calibration/mask_v1.X.9_im_sim.yaml # relative to sp_validation_repo # match_radius_deg: 0.0002 # RA/Dec pair-match radius, degrees - # w_cols: [none, w_iv] # weight schemes to compute in one run; "none" - # # (or null) = unit weights (#227); first entry is - # # the primary/headline result -- lead with "none" - # # for the fiducial unweighted estimator. The - # # deprecated scalar `w_col` is still accepted. - # pair_match: true # match +g/-g object-by-object (per-object cancellation) + # w_cols: [w_des] # must match the calibration's weight_scheme: des + # pair_match: false # full populations, branch-specific normalization; + # # modern products refuse symmetric pair weighting # n_bootstrap: 500 # bootstrap resamples for the errors # bootstrap_seed: 42 # seed for the bootstrap RNG (makes errors reproducible) diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index cc992391..b08df676 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -16,7 +16,8 @@ check at load rejects an unknown key (typo) and a missing science key (see The five simulations per grid are the reference ``1z2z`` (no input shear) plus the ``+/-`` shear pairs ``1p2z``/``1m2z`` (g1) and ``1z2p``/``1z2m`` (g2); the -m-bias estimator matches each to the reference by RA/Dec. +modern m-bias estimator uses full populations with separately normalized +branch weights (pair_match: false), matching the stored response. """ import os @@ -54,6 +55,8 @@ _OPERATIONAL_KEYS = { "n_smp", "extract_script", "calibrate_script", + "des_weight_source", + "des_weight_product", } # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { @@ -91,6 +94,18 @@ if _missing_structural: f"{_missing_structural} -- set them in the run config" ) +if "w_des" in IMSIM["w_cols"]: + if IMSIM["des_weight_source"] not in ("external", "self"): + raise ValueError("image_sims: des_weight_source must be external or self") + _product = IMSIM["des_weight_product"] + if IMSIM["des_weight_source"] == "external" and ( + not isinstance(_product, dict) or not _product.get("name") or not _product.get("path") + ): + raise ValueError( + "DES image sims require des_weight_product with name and path; " + "des_weight_source: self is diagnostic only" + ) + # --- container ------------------------------------------------------------ # Every compute rule carries ``container: SIF`` rather than inheriting a # module-level default: these rules are also included from the top-level @@ -256,8 +271,8 @@ rule im_init: and the raw SKiLLS image inputs. ``params_im_sim.py`` derives the field name from the directory basename, so - the same template serves every sim; ``config_mask.yaml`` and ``cfis`` are - symlinks the downstream calibration and merge steps read from cwd. + the same template serves every sim; ``config_mask.yaml`` is a staged copy + with the image-sim DES-table policy, and ``cfis`` is a symlink. ``input_tiles``/``input_exp`` are top-level symlinks to the raw SKiLLS tile and exposure images; ShapePipe's ``get_images_runner`` resolves them via @@ -271,6 +286,12 @@ rule im_init: # silently leaving stale params.py behind after a grammar change). template=PARAMS_TEMPLATE, mask_src=os.path.join(SPV_REPO, MASK_CONFIG), + weight_product=( + [IMSIM["des_weight_product"]["path"]] + if "w_des" in IMSIM["w_cols"] + and IMSIM["des_weight_source"] == "external" + and IMSIM["des_weight_product"] else [] + ), output: params=f"{GRIDS_BASE}/{{sim}}/params.py", mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", @@ -280,13 +301,22 @@ rule im_init: cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis", sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles", sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_exp", + source=IMSIM["des_weight_source"], + product_name=(IMSIM["des_weight_product"] or {}).get("name", "-"), + product_path=(IMSIM["des_weight_product"] or {}).get("path", "-"), + w_cols=IMSIM["w_cols"], + container: + SIF shell: # cfis / input_tiles / input_exp are stable read-only symlinks (used by # get_images, merge, extract); created here but not tracked as outputs, # which snakemake will not accept for a symlink/directory. "mkdir -p $(dirname {output.params}) && " "cp {input.template} {output.params} && " - "ln -sf {input.mask_src} {output.mask} && " + "{_ENV_PREFIX} python {SPV_REPO}/scripts/calibration/stage_image_sim_config.py " + "--input {input.mask_src:q} --output {output.mask:q} " + "--w-cols {params.w_cols} --source {params.source:q} " + "--product-name {params.product_name:q} --product-path {params.product_path:q} && " "ln -sfT {params.config_dir} {params.cfis} && " "ln -sfT {params.sim_tiles} {params.run_dir}/input_tiles && " "ln -sfT {params.sim_exp} {params.run_dir}/input_exp" From 50a30c62e1951e5a55431a60b9dcaaa3ef590f1f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 26 Oct 2026 19:51:22 +0100 Subject: [PATCH 25/36] Test response weights, sparse tables and shared-tile calibration by behaviour --- .../tests/test_additive_response.py | 35 ++ src/sp_validation/tests/test_calibration.py | 1 + src/sp_validation/tests/test_column_schema.py | 29 +- .../tests/test_configured_response.py | 126 ++++++ .../tests/test_consistent_response.py | 385 ++++++++++++++++++ src/sp_validation/tests/test_des_jackknife.py | 81 ++++ .../tests/test_des_table_independence.py | 45 ++ src/sp_validation/tests/test_external_des.py | 126 ++++++ .../tests/test_image_sims_jackknife.py | 100 +++++ .../tests/test_image_sims_weights.py | 67 +++ .../tests/test_inverse_variance.py | 87 ++++ .../tests/test_pair_response_jackknife.py | 48 +++ .../tests/test_recalibration_weights.py | 73 ++++ .../tests/test_response_diagnostics.py | 60 +++ .../tests/test_response_weights.py | 92 +++++ src/sp_validation/tests/test_sparse_des.py | 126 ++++++ 16 files changed, 1480 insertions(+), 1 deletion(-) create mode 100644 src/sp_validation/tests/test_additive_response.py create mode 100644 src/sp_validation/tests/test_configured_response.py create mode 100644 src/sp_validation/tests/test_consistent_response.py create mode 100644 src/sp_validation/tests/test_des_jackknife.py create mode 100644 src/sp_validation/tests/test_des_table_independence.py create mode 100644 src/sp_validation/tests/test_external_des.py create mode 100644 src/sp_validation/tests/test_image_sims_jackknife.py create mode 100644 src/sp_validation/tests/test_image_sims_weights.py create mode 100644 src/sp_validation/tests/test_inverse_variance.py create mode 100644 src/sp_validation/tests/test_pair_response_jackknife.py create mode 100644 src/sp_validation/tests/test_recalibration_weights.py create mode 100644 src/sp_validation/tests/test_response_diagnostics.py create mode 100644 src/sp_validation/tests/test_response_weights.py create mode 100644 src/sp_validation/tests/test_sparse_des.py diff --git a/src/sp_validation/tests/test_additive_response.py b/src/sp_validation/tests/test_additive_response.py new file mode 100644 index 00000000..0b58fa78 --- /dev/null +++ b/src/sp_validation/tests/test_additive_response.py @@ -0,0 +1,35 @@ +"""Additive-bias producer and consumer must agree on component units.""" + +from contextlib import nullcontext +from types import SimpleNamespace + +import numpy as np + +from sp_validation.cosmo_val.core import CosmologyValidation + + +def test_des_additive_bias_producer_to_consumer(): + obj = object.__new__(CosmologyValidation) + obj.versions = ["DES"] + obj.cc = { + "DES": { + "shear": dict( + e1_col="e1", e2_col="e2", w_col="w", R=1, R11="R11", R22="R22" + ) + } + } + data = dict( + e1=np.array([0.12, -0.04]), + e2=np.array([0.08, 0.16]), + w=np.ones(2), + R11=np.full(2, 0.8), + R22=np.full(2, 0.7), + ) + obj._results = { + "DES": SimpleNamespace(dat_shear=data, temporarily_read_data=nullcontext) + } + obj.print_start = obj.print_done = obj.print_magenta = lambda *args: None + obj.calculate_additive_bias() + np.testing.assert_allclose([obj.c1["DES"], obj.c2["DES"]], [0.04 / 0.8, 0.12 / 0.7]) + g = obj._calibrated_g("DES") + np.testing.assert_allclose(np.average(g, axis=1, weights=data["w"]), 0, atol=1e-16) diff --git a/src/sp_validation/tests/test_calibration.py b/src/sp_validation/tests/test_calibration.py index fbe78785..f6ab0af5 100644 --- a/src/sp_validation/tests/test_calibration.py +++ b/src/sp_validation/tests/test_calibration.py @@ -51,6 +51,7 @@ def make_fake_gal_metacal(R=R, g1=G1, g2=G2, w=W, mask=MASK): populated, so the test exercises the pure linear algebra and nothing else. """ return types.SimpleNamespace( + _legacy_response=True, # These literals characterize released legacy c/scatter. R=np.asarray(R, dtype=float), ns={ "g1": np.asarray(g1, dtype=float), diff --git a/src/sp_validation/tests/test_column_schema.py b/src/sp_validation/tests/test_column_schema.py index da073b80..51229544 100644 --- a/src/sp_validation/tests/test_column_schema.py +++ b/src/sp_validation/tests/test_column_schema.py @@ -39,7 +39,34 @@ "R", "R11", "R22", - # FITS header keywords + # FITS header keywords (response/c units, weighting and spatial products) + "CE_{}", + "CG_{}", + "CMUNITS", + "DESMERGE", + "DESNMIN", + "IVPOLICY", + "MCSTEP", + "MEANSUB", + "MIG_{}", + "N_{}", + "ORDER", + "RAPPLIED", + "RJ_{}{}", + "R_{}{}", # derived response-product matrix, not parent shape columns + "R_JK", # response-product FITS extension name + "RLEGACY", + "SW_{}", + "WGHTSCH", + "WPOLICY", + "WSNRCOL", + "WSIZDEF", + "WSOURCE", + "WPRODUCT", + "WPRODPTH", + "W_DES", # response-product FITS images, not parent catalogue columns + "W_COUNT", + "W_EDGES", "EXTNAME", "R_S11", "R_S12", diff --git a/src/sp_validation/tests/test_configured_response.py b/src/sp_validation/tests/test_configured_response.py new file mode 100644 index 00000000..650cb616 --- /dev/null +++ b/src/sp_validation/tests/test_configured_response.py @@ -0,0 +1,126 @@ +"""Global metacal response must use the weights of the shear statistic.""" + +from pathlib import Path + +import numpy as np +import yaml + +from sp_validation import calibration +from sp_validation.calibration import metacal + +GAMMA = np.array([0.02, -0.01]) +STEP = 0.01 +POPULATIONS = [(0.6, 1.0), (0.9, 3.0)] # (response, statistic weight) +N_PER_POP = 400 # Even: intrinsic ellipticities come in +/- pairs. + + +def _repository_root(): + root = Path(__file__).resolve().parents[3] + # A cross-check can import another checkout via PYTHONPATH while keeping + # this test file. Use that checkout's configuration as well as its code. + source_root = Path(calibration.__file__).resolve().parents[2] + if source_root != root and (source_root / "pyproject.toml").is_file(): + return source_root + return root + + +def _synthetic_catalogue(): + """ngmix-style catalogue with known responses and zero selection response.""" + rng = np.random.default_rng(1) + names = ["1M", "1P", "2M", "2P", "NOSHEAR"] + fields = [] + for name in names: + fields.extend( + [ + (f"NGMIX_FLAGS_{name}", "i4"), + (f"NGMIX_G1_{name}", "f8"), + (f"NGMIX_G2_{name}", "f8"), + (f"NGMIX_FLUX_{name}", "f8"), + (f"NGMIX_FLUX_ERR_{name}", "f8"), + (f"NGMIX_T_{name}", "f8"), + (f"NGMIX_T_ERR_{name}", "f8"), + (f"NGMIX_T_PSF_RECONV_{name}", "f8"), + ] + ) + fields += [("NGMIX_G1_ERR_NOSHEAR", "f8"), ("NGMIX_G2_ERR_NOSHEAR", "f8")] + ntot = N_PER_POP * len(POPULATIONS) + data = np.zeros(ntot, dtype=fields) + response = np.concatenate([np.full(N_PER_POP, r) for r, _ in POPULATIONS]) + w_des = np.concatenate([np.full(N_PER_POP, w) for _, w in POPULATIONS]) + intrinsic = np.empty((2, ntot)) + for k in range(len(POPULATIONS)): + half = rng.normal(0, 0.25, size=(2, N_PER_POP // 2)) + # Exact second moments give the target 1:3 weights in distinct cells. + r, target_w = POPULATIONS[k] + noise = r**2 / target_w - 0.5 * r**2 * np.sum(GAMMA**2) + half *= np.sqrt(noise / np.mean(half**2)) + intrinsic[:, k * N_PER_POP : (k + 1) * N_PER_POP] = np.concatenate( + [half, -half], axis=1 + ) + g_ns = intrinsic + response * GAMMA[:, None] + shifts = { + "NOSHEAR": (0.0, 0.0), + "1P": (STEP, 0.0), + "1M": (-STEP, 0.0), + "2P": (0.0, STEP), + "2M": (0.0, -STEP), + } + for name, (d1, d2) in shifts.items(): + data[f"NGMIX_G1_{name}"] = g_ns[0] + response * d1 + data[f"NGMIX_G2_{name}"] = g_ns[1] + response * d2 + data[f"NGMIX_FLUX_{name}"] = np.repeat([30.0, 100.0], N_PER_POP) + data[f"NGMIX_FLUX_ERR_{name}"] = 1.0 # SNR 50: inside all cuts. + data[f"NGMIX_T_{name}"] = 1.5 + data[f"NGMIX_T_PSF_RECONV_{name}"] = 1.0 # Size ratio inside all cuts. + data["NGMIX_G1_ERR_NOSHEAR"] = 0.01 + data["NGMIX_G2_ERR_NOSHEAR"] = 0.01 + return data, w_des + + +def test_w_des_weighted_shear_unbiased_with_committed_global_response_weight(): + """Protect multiplicative calibration of production w_des statistics. + + mask_v2.0.yaml sets the weight used to average the global response; + cosmo_val uses w_des with shear.R=1. A weighted shear estimator is unbiased + only when the applied response is averaged with the statistic's weights. + Intrinsic shapes cancel in pairs and selection response is zero here. + Responses 0.6/0.9 with w_des=1/3 give _w_des=0.825, while equal + measurement errors give _w_iv=0.75. The calibrated weighted mean must + equal gamma, not 1.1*gamma. + """ + root = _repository_root() + config = yaml.safe_load((root / "config/calibration/mask_v2.0.yaml").read_text())[ + "metacal" + ] + stat_config = yaml.safe_load((root / "cosmo_val/cat_config.yaml").read_text()) + # Check the production statistic's weight, rather than an unused template. + shear_config = stat_config["SP_v1.4.6.3"]["shear"] + assert shear_config["w_col"] == "w_des" + assert shear_config["R"] == 1.0 + data, w_des = _synthetic_catalogue() + gal = metacal( + data, + np.ones(len(data), bool), + snr_min=config["gal_snr_min"], + snr_max=config["gal_snr_max"], + rel_size_min=config["gal_rel_size_min"], + rel_size_max=config["gal_rel_size_max"], + size_corr_ell=config["gal_size_corr_ell"], + sigma_eps=config["sigma_eps_prior"], + weight_scheme=config["weight_scheme"], + ) + # The production W fit/evaluation must reproduce the target weights. + np.testing.assert_allclose(gal.ns["w_des"], w_des, rtol=1e-12) + np.testing.assert_allclose(np.diag(gal.R), 0.825, atol=1e-12) + assert len(gal.mask_dict["ns"]) == len(data) + np.testing.assert_allclose(gal.R_selection, 0, atol=1e-12) + g_corr, _, _, mask, _, _ = calibration.get_calibrated_m_c( + gal, additive_correction=False + ) + gamma_hat = np.array([np.average(g_corr[c], weights=w_des[mask]) for c in (0, 1)]) + assert np.allclose(gamma_hat, GAMMA, rtol=1e-6), ( + f"w_des-weighted shear biased: gamma_hat={gamma_hat}, gamma={GAMMA}, " + f"m={gamma_hat / GAMMA - 1}; global_R_weight=" + f"{config['weight_scheme']!r}, applied diag={np.diag(gal.R)}, " + "expected statistic response=0.825" + ) diff --git a/src/sp_validation/tests/test_consistent_response.py b/src/sp_validation/tests/test_consistent_response.py new file mode 100644 index 00000000..bc4db7eb --- /dev/null +++ b/src/sp_validation/tests/test_consistent_response.py @@ -0,0 +1,385 @@ +"""Analytic S1/case-d checks; no astronomical input required.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import calibration, io, response + + +def catalogue(n=80): + fields = [] + for b in ("NOSHEAR", "1P", "1M", "2P", "2M"): + fields += [ + (f"NGMIX_{k}_{b}", "f8") + for k in ( + "FLAGS", + "G1", + "G2", + "G1_ERR", + "G2_ERR", + "FLUX", + "FLUX_ERR", + "T", + "T_ERR", + "T_PSF_RECONV", + ) + ] + d = np.zeros(n, dtype=fields) + matrix = np.array([[0.8, 0.12], [-0.06, 0.9]]) + intrinsic = np.tile([[0.2, -0.2], [0.1, -0.1]], (1, n // 2)) + ce = matrix @ np.array([0.02, -0.01]) + np.array([0.004, -0.002]) + for b, shift in [ + ("NOSHEAR", [0, 0]), + ("1P", [0.01, 0]), + ("1M", [-0.01, 0]), + ("2P", [0, 0.01]), + ("2M", [0, -0.01]), + ]: + e = intrinsic + ce[:, None] + (matrix @ shift)[:, None] + for i in (1, 2): + d[f"NGMIX_G{i}_{b}"] = e[i - 1] + d[f"NGMIX_FLUX_{b}"] = np.repeat([30.0, 100.0], n // 2) + d[f"NGMIX_FLUX_ERR_{b}"] = 1 + d[f"NGMIX_T_{b}"] = 1 + d[f"NGMIX_T_PSF_RECONV_{b}"] = 1 + return d, matrix, ce + + +@pytest.mark.parametrize("scheme", ["des", "iv"]) +def test_full_matrix_c_units_and_sim_signal(scheme): + d, r, ce = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme=scheme, + size_corr_ell=False, + des_num_bins=2, + ) + np.testing.assert_allclose(gm.R, r, atol=1e-14) + np.testing.assert_allclose(gm.R_joint, r, atol=1e-14) + np.testing.assert_allclose(gm.c_e, ce, atol=1e-14) + np.testing.assert_allclose(gm.c_g, np.linalg.solve(r, ce), atol=1e-14) + sim, _, w, _, _, _ = calibration.get_calibrated_m_c(gm, False) + real, *_ = calibration.get_calibrated_m_c(gm, True) + np.testing.assert_allclose(np.average(real, weights=w, axis=1), 0, atol=1e-14) + np.testing.assert_allclose(np.average(sim, weights=w, axis=1), gm.c_g) + + +def test_fixed_weight_c_uses_final_weights(): + d, _, _ = catalogue() + d["NGMIX_G1_ERR_NOSHEAR"] = np.linspace(0.01, 0.8, len(d)) + gm = calibration.metacal( + d, np.ones(len(d), bool), weight_scheme="iv", size_corr_ell=False + ) + _, raw, w, _, ce, _ = calibration.get_calibrated_m_c(gm) + np.testing.assert_allclose(ce, np.average(raw, weights=w, axis=1)) + assert abs(ce[0] - raw[0].mean()) > 1e-5 + + +def fixture_response(): + e = np.array([[0.2, -0.1, 0.4, -0.2], [0.1, 0.3, -0.2, 0.2]]) + r = np.array([[0.8, 0.1], [-0.04, 0.9]]) + branches = {"ns": e} + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + branches[p] = e + 0.01 * r[:, j, None] + branches[m] = e - 0.01 * r[:, j, None] + masks = {b: np.arange(4) for b in branches} + rg = np.repeat(r[:, :, None], 4, axis=2) + w = {b: np.array([1.0, 2.0, 3.0, 4.0]) for b in branches} + return branches, masks, rg, w + + +def test_selection_separate_denominators(): + b, masks, rg, w = fixture_response() + masks["p1"] = np.array([0, 1, 2]) + masks["m1"] = np.array([1, 3]) + res = response.responses(b, masks, w, rg, 0.01) + expected = ( + np.average(b["ns"][:, masks["p1"]], weights=w["p1"][masks["p1"]], axis=1) + - np.average(b["ns"][:, masks["m1"]], weights=w["m1"][masks["m1"]], axis=1) + ) / 0.02 + np.testing.assert_allclose(res["selection"][:, 0], expected) + + +def test_pure_cell_migration_all_elements(): + b, masks, rg, _ = fixture_response() + # Fixed two-cell W. No cut crossing; all four selection elements nonzero. + table = ( + np.array([10.0, 50.0, 500.0]), + np.array([0.5, 3.0]), + np.array([[1.0], [4.0]]), + None, + ) + positions = { + "ns": [20, 20, 100, 100], + "p1": [100, 20, 100, 20], + "m1": [20, 100, 20, 100], + "p2": [100, 100, 20, 100], + "m2": [20, 20, 100, 20], + } + w = { + k: response.evaluate_w(table, np.array(s), np.ones(4)) + for k, s in positions.items() + } + res = response.responses(b, masks, w, rg, 0.01) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + expected = ( + np.average(b["ns"], weights=w[p], axis=1) + - np.average(b["ns"], weights=w[m], axis=1) + ) / 0.02 + np.testing.assert_allclose(res["selection"][:, j], expected) + frozen = response.responses(b, masks, {k: w["ns"] for k in w}, rg, 0.01) + np.testing.assert_allclose(frozen["selection"], 0, atol=1e-14) + assert np.all(abs(res["selection"]) > 0.1) + np.testing.assert_allclose(res["joint"], res["total"], atol=1e-14) + scaled = response.responses(b, masks, {k: 17 * v for k, v in w.items()}, rg, 0.01) + np.testing.assert_allclose(scaled["total"], res["total"]) + + +def test_w_moments_edges_empty_invalid_and_new_rows(): + table = response.fit_w( + np.array([10.0, 50.0, 100.0]), + np.ones(3), + np.array([[0.2, 0.2, 0.4], [0.2, 0.2, 0.4]]), + np.array([0.8, 0.8, 0.8]), + np.array([10.0, 50.0, 500.0]), + np.array([0.5, 2.0, 3.0]), + ) + np.testing.assert_allclose(table[2], [[16.0, 0], [4.0, 0]]) + s = np.array([1.0, 50.0, 51.0, 1000.0, np.nan, 20.0]) + size = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 2.5]) + np.testing.assert_allclose( + response.evaluate_w(table, s, size), [16, 16, 4, 4, 0, 0] + ) + + +def test_spatial_jackknife_shared_selection_and_covariance(): + b, masks, rg, w = fixture_response() + groups = np.array(["a", "b", "c", "d"]) + masks["p1"] = np.array([0, 1, 2]) + masks["m1"] = np.array([1, 2, 3]) + labels, estimates, cov = response.jackknife(b, masks, w, rg, 0.01, groups) + brute = [response.responses(b, masks, w, rg, 0.01, groups != g) for g in labels] + for key in estimates: + np.testing.assert_allclose(estimates[key], [x[key] for x in brute]) + v = np.concatenate([estimates[k].reshape(4, -1) for k in estimates], axis=1) + dv = v - v.mean(axis=0) + np.testing.assert_allclose(cov, 0.75 * dv.T @ dv) + assert cov[4, 4] > 0 + + +def test_end_to_end_cell_migration_and_sheared_only_row(): + d, _, _ = catalogue() + for branch in ("NOSHEAR", "1P", "1M", "2P", "2M"): + d[f"NGMIX_G1_{branch}"][40:] += np.tile([0.15, -0.15], 20) + d["NGMIX_FLUX_1P"][:40:2] = 100 + d["NGMIX_FLUX_1M"][1:40:2] = 100 + # A common-parent row absent from S_ns still gets a defined branch W. + d["NGMIX_FLUX_NOSHEAR"][0] = 9 + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + assert 0 not in gm.mask_dict["ns"] + assert 0 in gm.mask_dict["p1"] + assert gm.branch_weights["p1"][0] > 0 + assert gm.branch_migration["p1"] > 0 + manual = [] + for key in ("p1", "m1"): + idx = gm.mask_dict[key] + manual.append( + np.average( + np.array([gm.ns["g1"][idx], gm.ns["g2"][idx]]), + weights=gm.branch_weights[key][idx], + axis=1, + ) + ) + np.testing.assert_allclose(gm.R_selection[:, 0], (manual[0] - manual[1]) / 0.02) + frozen = {b: gm.branch_weights["ns"] for b in response.BRANCHES} + res = response.responses(gm.branches, gm.mask_dict, frozen, gm.R_shear, 0.01) + assert abs(res["selection"][0, 0] - gm.R_selection[0, 0]) > 0.1 + + +def test_coherent_spatial_errors_exceed_independent_object_errors(): + d, _, _ = catalogue() + groups = np.repeat(np.arange(4), 20) + for comp in (1, 2): + for sign, step in (("P", 0.01), ("M", -0.01)): + d[f"NGMIX_G{comp}_{comp}{sign}"] += step * groups * 0.1 + gm = calibration.metacal( + d, np.ones(len(d), bool), weight_scheme="iv", size_corr_ell=False + ) + _, _, spatial = gm.response_jackknife(groups) + _, _, objects = gm.response_jackknife(np.arange(len(d))) + assert spatial[0, 0] > 10 * objects[0, 0] + + +def test_mismatched_config_and_recalibration_refusal(tmp_path): + d, _, _ = catalogue() + with pytest.raises(ValueError, match="disagrees"): + calibration.metacal(d, np.ones(len(d), bool), weight_scheme="iv", w_col="w_des") + hdr = fits.Header({"R_S11": 0.1, "R_S12": 0.0, "R_S21": 0.0, "R_S22": 0.1}) + path = tmp_path / "insufficient.fits" + fits.HDUList([fits.PrimaryHDU(header=hdr), fits.BinTableHDU()]).writeto(path) + with pytest.raises(ValueError, match="parent five-branch"): + calibration.get_calibrate_e_from_cat(path) + + +def test_recalibration_requires_unchanged_population_with_nonzero_selection(tmp_path): + path = tmp_path / "compatible.fits" + truth = np.repeat([0.02, -0.02], 4) + raw = 0.9 * truth + zeros = np.zeros(8) + cols = [ + fits.Column(name="e1_uncal", array=raw, format="D"), + fits.Column(name="e2_uncal", array=zeros, format="D"), + fits.Column(name="w_des", array=np.ones(8), format="D"), + ] + for i in (1, 2): + for j in (1, 2): + cols.append( + fits.Column( + name=f"R_g{i}{j}", + array=np.full(8, 0.8 if i == j else 0), + format="D", + ) + ) + hdr = fits.Header( + { + "R_S11": 0.1, + "R_S12": 0.0, + "R_S21": 0.0, + "R_S22": 0.1, + "WGHTSCH": "des", + "N_ns": 8, + "SW_ns": 8.0, + } + ) + fits.HDUList( + [fits.PrimaryHDU(header=hdr), fits.BinTableHDU.from_columns(cols)] + ).writeto(path) + g1, _ = calibration.get_calibrate_e_from_cat(path) + np.testing.assert_allclose(g1, truth) + with fits.open(path, mode="update") as hdus: + hdus[0].header["N_ns"] = 9 + with pytest.raises(ValueError, match="population/weights"): + calibration.get_calibrate_e_from_cat(path) + + +def test_downstream_c_is_calibrated_not_raw(): + from types import SimpleNamespace + + from sp_validation.cosmo_val.core import CosmologyValidation + + e = np.array([[0.12, -0.04], [0.08, 0.16]]) + cg = np.array([0.02, -0.03]) + obj = SimpleNamespace( + _read_shear_cols=lambda *args: e, + _component_responses=lambda ver: (0.8, 0.8), + cc={"external": {"shear": {"R": 0.8}}}, + c1={"external": cg[0]}, + c2={"external": cg[1]}, + ) + result = CosmologyValidation._calibrated_g(obj, "external") + np.testing.assert_allclose(result, e / 0.8 - cg[:, None]) + assert not np.allclose(result, (e - cg[:, None]) / 0.8) + + +@pytest.mark.parametrize("adapter", ["xi", "pseudo_cl", "rho_tau", "leakage"]) +def test_calibrated_matrix_shear_passes_reader_adapters_once( + tmp_path, monkeypatch, adapter +): + from types import SimpleNamespace + + from sp_validation.cosmo_val import core + from sp_validation.rho_tau import _CatalogueLoader + + d, _, _ = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + g, _, w, _, _, _ = calibration.get_calibrated_m_c(gm, False) + path = tmp_path / "matrix_calibrated.fits" + cols = [ + fits.Column(name="e1", array=g[0], format="D"), + fits.Column(name="e2", array=g[1], format="D"), + fits.Column(name="w_des", array=w, format="D"), + ] + fits.HDUList([fits.PrimaryHDU(), fits.BinTableHDU.from_columns(cols)]).writeto(path) + calibration.save_response_product(path, gm) + entry = dict(path=path, w_col="w_des", e1_col="e1", e2_col="e2", R=1.0) + + def read(block): + if adapter == "rho_tau": + loader = _CatalogueLoader({"shear": block}, {}) + filename = loader("shear") # ordinary FITS is passed through + with fits.open(filename) as hdus: + return np.array([hdus[1].data["e1"], hdus[1].data["e2"]]) + if adapter == "leakage": + obj = core._LeakageObject.__new__(core._LeakageObject) + obj.entries = {"shear": block} + obj.read_data() + return np.array([obj._dat["e1"], obj._dat["e2"]]) + if adapter == "xi": + monkeypatch.setattr(core.leakage, "cut_data", lambda data, *args: data) + obj = core._LeakageScale.__new__(core._LeakageScale) + obj.entries = {"shear": block} + obj._params = {"cut": None, "verbose": False} + obj.read_data(shear=True, psf=False) + return np.array([obj.dat_shear["e1"], obj.dat_shear["e2"]]) + obj = SimpleNamespace( + cc={"SP": {"shear": block}}, + results={"SP": SimpleNamespace(dat_shear=io.open_entry(block))}, + ) + return np.array( + core.CosmologyValidation._read_shear_cols(obj, "SP", "e1_col", "e2_col") + ) + + np.testing.assert_allclose(read(entry), g) + for wrong in ( + dict(entry, w_col="w_iv"), + dict(entry, R=0.8), + dict(entry, R11="R_g11"), + ): + with pytest.raises(ValueError): + read(wrong) + + +def test_persistence_and_downstream_no_double_R(tmp_path): + d, _, _ = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + path = tmp_path / "cut.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.BinTableHDU.from_columns( + [fits.Column(name="w_des", array=gm.ns["w"], format="D")] + ), + ] + ).writeto(path) + calibration.save_response_product(path, gm, np.arange(len(d)) % 4) + with fits.open(path) as hdus: + assert hdus["R_COV"].data.shape == (20, 20) + assert hdus[0].header["RAPPLIED"] + assert hdus[0].header["WGHTSCH"] == "des" + for entry in ( + {"path": path, "w_col": "w_iv", "R": 1}, + {"path": path, "w_col": "w_des", "R": 0.8}, + ): + with pytest.raises(ValueError): + io.open_entry(entry) + assert len(io.open_entry({"path": path, "w_col": "w_des", "R": 1})) == len(d) diff --git a/src/sp_validation/tests/test_des_jackknife.py b/src/sp_validation/tests/test_des_jackknife.py new file mode 100644 index 00000000..c19898d7 --- /dev/null +++ b/src/sp_validation/tests/test_des_jackknife.py @@ -0,0 +1,81 @@ +"""Refitted cell sums versus the retained-row reference, including empty cells.""" + +import numpy as np +import pytest + +from sp_validation import calibration, response +from sp_validation.des_jackknife import accumulate +from sp_validation.tests.test_consistent_response import catalogue + + +@pytest.mark.parametrize("min_count", [0, 3, 10, 1000]) +def test_refit_cell_sums_match_rescans_with_empty_cell(min_count): + data, _, _ = catalogue() + # Only group zero trains the upper-right cell. Other artificial branches + # migrate into it, so its newly pooled weight affects the retained response. + for suffix in ("NOSHEAR", "1P", "1M", "2P", "2M"): + data[f"NGMIX_FLUX_{suffix}"] = 30 + data[f"NGMIX_T_{suffix}"] = 0.7 + data[f"NGMIX_FLUX_{suffix}"][:3] = 200 + data[f"NGMIX_T_{suffix}"][:3] = 2 + data[f"NGMIX_G1_{suffix}"][:20] += 0.13 + data["NGMIX_FLUX_1P"][20:24] = 200 + data["NGMIX_T_1P"][20:24] = 2 + data["NGMIX_FLUX_2M"][30:35] = 200 + data["NGMIX_T_2M"][30:35] = 2 + # Sheared-only and no-shear-only objects exercise mask/rg alignment. + data["NGMIX_FLUX_NOSHEAR"][50] = 9 + data["NGMIX_FLUX_1M"][51] = 9 + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + des_min_count=min_count, + ) + groups = np.repeat(np.arange(4), 20) + acc = accumulate(gm, groups, chunk_size=7) + assert gm.W[3][1, 1] == 3 + assert acc.training[0, :, 3].sum() - acc.training[0, 0, 3] == 0 + for k, label in enumerate(acc.labels): + keep = groups != label + idx = gm.mask_dict["ns"] + retained = keep[idx] + table = response.fit_w( + (gm.ns["flux"] / gm.ns["flux_err"])[idx][retained], + (gm.ns["T"] / gm.ns["Tpsf"])[idx][retained], + np.array(gm.branches["ns"])[:, idx][:, retained], + (0.5 * (gm.R11 + gm.R22))[retained], + gm.W[0], + gm.W[1], + min_count=min_count, + ) + np.testing.assert_allclose(acc.deleted_weight(k), table[2], atol=1e-12, rtol=0) + if k == 0: + assert (acc.deleted_weight(k)[1, 1] > 0) == (min_count > 0) + labels, fast, cov = gm.response_jackknife(groups, refit_w=True) + ref_labels, ref, ref_cov = gm.response_jackknife( + groups, refit_w=True, reference=True + ) + np.testing.assert_array_equal(labels, ref_labels) + for key in fast: + np.testing.assert_allclose(fast[key], ref[key], atol=1e-12, rtol=0) + np.testing.assert_allclose( + acc.jackknife()[1][key], ref[key], atol=1e-12, rtol=0 + ) + np.testing.assert_allclose(cov, ref_cov, atol=1e-12, rtol=0) + + +def test_refit_group_validation(): + data, _, _ = catalogue() + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + ) + with pytest.raises(ValueError, match="align"): + gm.response_jackknife(np.arange(10), refit_w=True) + with pytest.raises(ValueError, match="at least two"): + gm.response_jackknife(np.zeros(len(data)), refit_w=True) diff --git a/src/sp_validation/tests/test_des_table_independence.py b/src/sp_validation/tests/test_des_table_independence.py new file mode 100644 index 00000000..1087d235 --- /dev/null +++ b/src/sp_validation/tests/test_des_table_independence.py @@ -0,0 +1,45 @@ +"""A DES table fitted on the measured population suppresses its shear signal.""" + +from itertools import product + +import numpy as np +import pytest + +from sp_validation import response + + +@pytest.mark.parametrize("occupancy", [3, 4, 8]) +def test_self_fitted_table_bias_is_inverse_cell_occupancy(occupancy): + # Enumerate every Rademacher sample instead of drawing Monte Carlo trials. + # Each sample is one cell; duplicate it for two independent training signs. + signs = np.array(list(product((-1.0, 1.0), repeat=occupancy))) + x = np.repeat(signs, 2, axis=0) * 0.25 + e = np.stack((x, np.full_like(x, 0.25))) + independent = e.copy() + independent[0] = np.tile([-0.25, 0.25], len(signs))[:, None] + cells = len(x) + snr = np.repeat(np.arange(cells) + 0.5, occupancy) + size = np.ones(len(snr)) + edges_s = np.arange(cells + 1) + edges_r = np.array([0.0, 2.0]) + step = 1e-5 + biases = [] + for training in (e, independent): + means = [] + for shear in (-step, step): + shifted = training.copy() + shifted[0] += shear + table = response.fit_w( + snr, + size, + shifted.reshape(2, -1), + np.ones(len(snr)), + edges_s, + edges_r, + min_count=3, + ) + weights = response.evaluate_w(table, snr, size) + means.append(np.average((e[0] + shear).ravel(), weights=weights)) + biases.append((means[1] - means[0]) / (2 * step) - 1) + assert biases[0] == pytest.approx(-1 / occupancy, abs=3e-9) + assert biases[1] == pytest.approx(0, abs=3e-9) diff --git a/src/sp_validation/tests/test_external_des.py b/src/sp_validation/tests/test_external_des.py new file mode 100644 index 00000000..92112342 --- /dev/null +++ b/src/sp_validation/tests/test_external_des.py @@ -0,0 +1,126 @@ +"""Independent data-table policy for image-simulation calibration.""" + +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import yaml +from astropy.io import fits + +from sp_validation import calibration, catalog_builders, response +from sp_validation.image_sim_config import calibration_config +from sp_validation.tests.test_consistent_response import catalogue + + +def make_product(tmp_path): + data, _, _ = catalogue() + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + path = tmp_path / "data-calibration.fits" + fits.HDUList([fits.PrimaryHDU()]).writeto(path) + calibration.save_response_product(path, gm) + return dict(name="synthetic-data-calibration", path=str(path)), gm + + +def test_external_table_never_trains_on_sim_shapes_and_uses_fixed_jk(tmp_path): + product, source = make_product(tmp_path) + data, _, _ = catalogue() + for suffix in ("NOSHEAR", "1P", "1M", "2P", "2M"): + data[f"NGMIX_G1_{suffix}"][:20] += 0.3 + kwargs = dict(weight_scheme="des", size_corr_ell=False, des_weight_product=product) + gm = calibration.metacal(data, np.ones(len(data), bool), **kwargs) + # Product edges win, even when the sim's configured bin count differs. + np.testing.assert_array_equal(gm.W[2], source.W[2]) + for b in response.BRANCHES: + np.testing.assert_array_equal(gm.branch_weights[b], source.branch_weights[b]) + groups = np.repeat(np.arange(4), 20) + labels, jk, cov = gm.response_jackknife(groups, refit_w=True) + fixed = gm.response_jackknife(groups, refit_w=False) + np.testing.assert_allclose(cov, fixed[2]) + for key in jk: + np.testing.assert_allclose(jk[key], fixed[1][key]) + for j, label in enumerate(labels): + keep = groups != label + retained = calibration.metacal(data[keep], np.ones(keep.sum(), bool), **kwargs) + np.testing.assert_allclose(jk["c_g"][j], retained.c_g, atol=1e-13) + assert gm.refit_weights(keep) is gm.branch_weights + path = tmp_path / "sim.fits" + header = fits.Header() + # Exercise the catalogue driver's config-to-header path with the named + # product mapping, before attaching the response and weight extensions. + catalog_builders.CalibrateCat().add_params_to_FITS_header( + header, cm=dict(weight_scheme="des", des_weight_product=product) + ) + fits.HDUList([fits.PrimaryHDU(header=header)]).writeto(path) + calibration.save_response_product(path, gm, groups) + with fits.open(path) as f: + assert f[0].header["WPOLICY"] == "fixed-external" + assert f[0].header["WSOURCE"] == "external" + assert f[0].header["WPRODUCT"] == product["name"] + np.testing.assert_allclose(f["R_JK"].data["c_g1"], jk["c_g"][:, 0]) + np.testing.assert_array_equal(f["W_DES"].data, source.W[2]) + + +def test_reject_product_without_verified_coordinates(tmp_path): + product, _ = make_product(tmp_path) + with fits.open(product["path"], mode="update") as f: + del f[0].header["WSIZDEF"] + with pytest.raises(ValueError, match="coordinate metadata"): + calibration.load_des_weight_product(product) + + +def test_staging_replaces_old_symlink_without_editing_shared_config(tmp_path): + original = "metacal:\n global_R_weight: null\n additive_correction: true\n" + source = tmp_path / "mask.yaml" + source.write_text(original) + staged = tmp_path / "config_mask.yaml" + staged.symlink_to(source) + script = ( + Path(__file__).resolve().parents[3] + / "scripts/calibration/stage_image_sim_config.py" + ) + subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(source), + "--output", + str(staged), + "--w-cols", + "w_des", + "--product-name", + "data", + "--product-path", + "/data.fits", + ], + check=True, + ) + assert not staged.is_symlink() + assert source.read_text() == original + cm = yaml.safe_load(staged.read_text())["metacal"] + assert cm["des_weight_product"] == dict(name="data", path="/data.fits") + assert cm["weight_scheme"] == "des" + + +def test_image_sim_default_requires_named_external_product(): + original = dict(metacal=dict(global_R_weight=None, additive_correction=True)) + with pytest.raises(ValueError, match="require des_weight_product"): + calibration_config(original, ["w_des"]) + product = dict(name="data-release", path="/calibration/data.fits") + staged = calibration_config(original, ["w_des"], product=product) + assert staged["metacal"]["des_weight_product"] == product + assert staged["metacal"]["weight_scheme"] == "des" + assert not staged["metacal"]["additive_correction"] + assert "global_R_weight" not in staged["metacal"] + assert "weight_scheme" not in original["metacal"] + diagnostic = calibration_config(staged, ["w_des"], source="self") + assert "des_weight_product" not in diagnostic["metacal"] + assert calibration_config(original, ["w_iv"]) == original diff --git a/src/sp_validation/tests/test_image_sims_jackknife.py b/src/sp_validation/tests/test_image_sims_jackknife.py new file mode 100644 index 00000000..171f260c --- /dev/null +++ b/src/sp_validation/tests/test_image_sims_jackknife.py @@ -0,0 +1,100 @@ +"""Shared-tile, response-recalibrating m/c covariance.""" + +import numpy as np +import pytest + +from sp_validation.image_sims import ImageSimMBias + + +@pytest.mark.parametrize("refit", [False, True]) +def test_shared_tile_deletions_recalibrate_both_branches(refit): + estimator = ImageSimMBias( + dict( + shear_amplitude=0.025, + match_radius_deg=0.0002, + pair_match=False, + w_cols=["w_des"], + n_bootstrap=5, + bootstrap_seed=42, + ) + ) + tiles = np.array(["a", "a", "b", "c"]) + labels = np.array(["c", "a", "b"]) + expected = [] + fixed_response = [] + for p, m, comp in estimator.pairs: + for name, sign in ((p, 1), (m, -1)): + raw = np.array([[0.1, -0.2, 0.03, 0.2], [0.2, -0.1, 0.3, -0.2]]) + raw[comp] += sign * 0.025 + # Off-diagonal terms exercise matrix orientation and coupling. + response = np.array([[0.8, 0.03], [-0.02, 0.7]]) + deleted_R = np.array( + [response + np.eye(2) * sign * x for x in (0.04, -0.02, 0.01)] + ) + g = np.linalg.solve(response, raw) + estimator.cats[name] = dict( + raw=raw, + e1=g[0], + e2=g[1], + tiles=tiles, + jk_labels=labels.copy(), + jk_R=deleted_R, + w={"w_des": np.array([1.0, 2.0, 3.0, 4.0])}, + weight_scheme="des", + W_policy="refit" if refit else "fixed", + jk_g=np.array( + [ + np.linalg.solve( + deleted_R[j], + np.average( + raw[:, tiles != label], + axis=1, + weights=np.array([1.0, 2.0, 3.0, 4.0])[tiles != label] + ** 2, + ), + ) + for j, label in enumerate(labels) + ] + ), + ) + draws, held_R = [], [] + for label in sorted(labels): + means, fixed = [], [] + for name in (p, m): + cat = estimator.cats[name] + keep = tiles != label + mean = np.average( + cat["raw"][:, keep], axis=1, weights=cat["w"]["w_des"][keep] + ) + index = list(labels).index(label) + means.append( + cat["jk_g"][index, comp] + if refit + else np.linalg.solve(cat["jk_R"][index], mean)[comp] + ) + fixed.append( + np.average( + cat[f"e{comp + 1}"][keep], weights=cat["w"]["w_des"][keep] + ) + ) + draws.append([(means[0] - means[1]) / 0.05 - 1, (means[0] + means[1]) / 2]) + held_R.append((fixed[0] - fixed[1]) / 0.05 - 1) + expected.extend(np.array(draws).T) + fixed_response.append(held_R) + result = estimator.run(False) + expected = np.array(expected).T + delta = expected - expected.mean(axis=0) + covariance = 2 / 3 * delta.T @ delta + np.testing.assert_allclose(result["mc_jackknife"], expected) + np.testing.assert_allclose(result["mc_covariance"], covariance) + np.testing.assert_allclose( + [result[k + "_err"] for k in result["covariance_order"]], + np.sqrt(covariance.diagonal()), + ) + assert abs(covariance[0, 1]) > 1e-6 + assert not np.allclose(expected[:, 0], fixed_response[0]) + assert result["uncertainty"] == "shared_tile_response_jackknife" + assert "conditional_object_bootstrap_fixed_response" not in result + estimator.cats[estimator.pairs[0][1]]["jk_labels"][0] = "d" + with pytest.raises(ValueError, match="identical parent"): + estimator.run(False) diff --git a/src/sp_validation/tests/test_image_sims_weights.py b/src/sp_validation/tests/test_image_sims_weights.py new file mode 100644 index 00000000..b2ba74df --- /dev/null +++ b/src/sp_validation/tests/test_image_sims_weights.py @@ -0,0 +1,67 @@ +"""Image-simulation numerator weights and paired-estimator restrictions.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation.image_sims import ImageSimMBias + + +def test_modern_headline_uses_branch_weights(tmp_path): + """Symmetric weights remove a real numerator contribution on matched rows.""" + for branch, sign in (("plus", 1), ("minus", -1)): + directory = tmp_path / f"{branch}_grid_1" + directory.mkdir() + arrays = dict( + RA=[30.0, 31.0], + Dec=[0.0, 0.0], + e1=np.array([-0.1, 0.1]) + sign * 0.02, + e2=[0.0, 0.0], + w_des=[1.0, 3.0] if sign == 1 else [3.0, 1.0], + ) + arrays["e1_uncal"] = arrays["e1"] + arrays["e2_uncal"] = arrays["e2"] + table = fits.BinTableHDU.from_columns( + [fits.Column(name=k, array=v, format="D") for k, v in arrays.items()] + + [fits.Column(name="TILE_ID", array=["a", "b"], format="1A")] + ) + jk = fits.BinTableHDU.from_columns( + [ + fits.Column(name="TILE_ID", array=["b", "a"], format="1A"), + *[ + fits.Column(name=f"R_{i}{j}", array=[float(i == j)] * 2, format="D") + for i in (1, 2) + for j in (1, 2) + ], + ], + name="R_JK", + ) + primary = fits.PrimaryHDU() + primary.header["WGHTSCH"] = "des" + primary.header["RAPPLIED"] = True + fits.HDUList([primary, table, jk]).writeto(directory / "cat.fits") + cfg = dict( + grids_dir=str(tmp_path), + num=1, + catalog_name="cat.fits", + shear_amplitude=0.02, + match_radius_deg=0.0002, + w_cols=["w_des"], + n_bootstrap=10, + bootstrap_seed=42, + pair_match=False, + branches=["plus", "minus"], + pairs=[dict(plus="plus", minus="minus", component=0)], + ) + estimator = ImageSimMBias(cfg) + estimator.load_catalogs(False) + result = estimator.run(False) + assert result["m1"] == pytest.approx(2.5) + # All objects match, but symmetric weighting would yield m=0. + symmetric_m = ( + np.mean((estimator.cats["plus"]["e1"] - estimator.cats["minus"]["e1"]) / 0.04) + - 1 + ) + assert symmetric_m == pytest.approx(0) + with pytest.raises(ValueError, match="pair_match=False"): + ImageSimMBias({**cfg, "pair_match": True}).load_catalogs(False) diff --git a/src/sp_validation/tests/test_inverse_variance.py b/src/sp_validation/tests/test_inverse_variance.py new file mode 100644 index 00000000..97054f48 --- /dev/null +++ b/src/sp_validation/tests/test_inverse_variance.py @@ -0,0 +1,87 @@ +"""ShapePipe G*_ERR columns store standard deviations, not variances.""" + +import numpy as np +import pytest + +from sp_validation.calibration import metacal + + +def _catalogue(): + fields = [] + for shear in ("NOSHEAR", "1M", "1P", "2M", "2P"): + fields.extend( + (f"NGMIX_{name}_{shear}", "f8") + for name in ( + "FLAGS", + "G1", + "G2", + "G1_ERR", + "G2_ERR", + "FLUX", + "FLUX_ERR", + "T", + "T_ERR", + "T_PSF_RECONV", + ) + ) + data = np.zeros(2, dtype=fields) + for shear in ("NOSHEAR", "1M", "1P", "2M", "2P"): + data[f"NGMIX_FLUX_{shear}"] = 50.0 + data[f"NGMIX_FLUX_ERR_{shear}"] = 1.0 + data[f"NGMIX_T_{shear}"] = 1.0 + data[f"NGMIX_T_PSF_RECONV_{shear}"] = 1.0 + # ERR columns store sqrt(diag(g_cov)), not diag(g_cov). + for shear in ("NOSHEAR", "1M", "1P", "2M", "2P"): + data[f"NGMIX_G1_ERR_{shear}"] = [0.05, 0.20] + data[f"NGMIX_G2_ERR_{shear}"] = [0.10, 0.30] + # Exactly diagonal per-object shear responses 0.6 I and 1.0 I. + for component in (1, 2): + data[f"NGMIX_G{component}_{component}P"] = [0.006, 0.010] + data[f"NGMIX_G{component}_{component}M"] = [-0.006, -0.010] + return data + + +@pytest.mark.parametrize("mask", [None, np.array([False, True])], ids=["all", "masked"]) +def test_iv_weights_square_both_shape_error_standard_deviations(mask): + """Protect weights from adding standard deviations to intrinsic variance. + + ShapePipe stores G1_ERR/G2_ERR as square roots of covariance diagonals. + For sigma_eps=0.34 and errors (0.05,0.10)/(0.20,0.30), the variances are + 2*0.34**2+0.05**2+0.10**2=0.2437 and + 2*0.34**2+0.20**2+0.30**2=0.3612. Their reciprocals are the correct + inverse-variance weights, including after masking. + """ + _, _, actual = metacal.get_variance_ivweights(_catalogue(), 0.34, mask=mask) + expected = 1.0 / np.array([0.2437, 0.3612]) + if mask is not None: + expected = expected[mask] + np.testing.assert_allclose(actual, expected, rtol=1e-12, atol=0) + + +def test_returned_covariance_diagonals_are_variances(): + c11, c22, _ = metacal.get_variance_ivweights(_catalogue(), 0.34) + np.testing.assert_allclose(c11, np.array([0.05, 0.20]) ** 2) + np.testing.assert_allclose(c22, np.array([0.10, 0.30]) ** 2) + + +def test_global_shear_response_uses_squared_shape_error_weights(): + """Protect the applied response from the same dimensional weight error. + + Both objects pass every sheared selection, so selection response is zero. + Their finite-difference shear responses are 0.6 I and 1.0 I. With + global_R_weight='w', the correct diagonal response is + (0.6/0.2437+1/0.3612)/(1/0.2437+1/0.3612), using intrinsic plus + measurement variances, rather than standard deviations. + """ + result = metacal( + _catalogue(), + np.ones(2, dtype=bool), + sigma_eps=0.34, + global_R_weight="w", + size_corr_ell=False, + ) + np.testing.assert_allclose(result.R_selection, np.zeros((2, 2)), atol=1e-14) + expected_scalar = (0.6 * 0.3612 + 1.0 * 0.2437) / (0.3612 + 0.2437) + np.testing.assert_allclose( + result.R, np.eye(2) * expected_scalar, rtol=1e-12, atol=1e-14 + ) diff --git a/src/sp_validation/tests/test_pair_response_jackknife.py b/src/sp_validation/tests/test_pair_response_jackknife.py new file mode 100644 index 00000000..37273701 --- /dev/null +++ b/src/sp_validation/tests/test_pair_response_jackknife.py @@ -0,0 +1,48 @@ +"""Stage-1 paired/global covariance, checked against explicit pair sums.""" + +import numpy as np +import pytest + +from sp_validation.response_diagnostics import scalar_pair_response + + +@pytest.mark.parametrize("varying", [False, True]) +def test_scalar_response_shared_patch_deletions(varying): + ra = np.arange(12, dtype=float) * 0.01 + dec = np.zeros(12) + w = np.arange(12, dtype=float) + 1 + r = np.linspace(0.5, 1.1, 12) if varying else np.full(12, 0.8) + patch = np.repeat(np.arange(4), 3) + measured = scalar_pair_response( + ra, + dec, + w, + r, + patch=patch, + min_sep=0.1, + max_sep=10, + nbins=1, + bin_slop=0, + ) + + def brute(keep): + ii, jj = np.triu_indices(len(ra), 1) + selected = keep[ii] & keep[jj] + ww = w[ii[selected]] * w[jj[selected]] + pair = np.average(r[ii[selected]] * r[jj[selected]], weights=ww) + return pair / np.average(r[keep], weights=w[keep]) ** 2 - 1 + + expected = np.array([brute(patch != j) for j in range(4)]) + # TreeCorr scalar fields are stored at float32 precision. + np.testing.assert_allclose( + measured["fraction"], brute(np.ones(12, bool)), atol=1e-7 + ) + np.testing.assert_allclose(measured["deleted"][:, 0], expected, atol=1e-7) + np.testing.assert_allclose( + measured["error"][0], + np.sqrt(0.75 * np.sum((expected - expected.mean()) ** 2)), + atol=1e-7, + ) + if not varying: + np.testing.assert_allclose(measured["fraction"], 0, atol=1e-7) + np.testing.assert_allclose(measured["error"], 0, atol=1e-7) diff --git a/src/sp_validation/tests/test_recalibration_weights.py b/src/sp_validation/tests/test_recalibration_weights.py new file mode 100644 index 00000000..84453bda --- /dev/null +++ b/src/sp_validation/tests/test_recalibration_weights.py @@ -0,0 +1,73 @@ +"""Recalibration must use the statistic's weights for both response and bias.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation.calibration import get_calibrate_e_from_cat + +G_TRUE = 0.02 + + +def _write_catalogue(path, weight_type): + # Opposite weight preferences: (response, w_des, w_iv). + # Intrinsic +/- pairs and patches +/-g make the global additive bias zero. + response, w_des, w_iv, intrinsic, shear = [], [], [], [], [] + for patch_shear in (+G_TRUE, -G_TRUE): + for r, wd, wi in ((0.5, 1.0, 3.0), (1.5, 3.0, 1.0)): + for e in (+0.1, -0.1): + response.append(r) + w_des.append(wd) + w_iv.append(wi) + intrinsic.append(e) + shear.append(patch_shear) + response, w_des, w_iv, intrinsic, shear = map( + np.array, (response, w_des, w_iv, intrinsic, shear) + ) + zeros = np.zeros_like(response) + columns = [ + fits.Column(name="e1_uncal", array=response * shear + intrinsic, format="D"), + fits.Column(name="e2_uncal", array=zeros, format="D"), + fits.Column(name="w_des", array=w_des, format="D"), + fits.Column(name="w_iv", array=w_iv, format="D"), + fits.Column(name="R_g11", array=response, format="D"), + fits.Column(name="R_g12", array=zeros, format="D"), + fits.Column(name="R_g21", array=zeros, format="D"), + fits.Column(name="R_g22", array=response, format="D"), + ] + # The response is valid for this unchanged population and weight identity. + header = fits.Header( + { + "WGHTSCH": weight_type, + "N_ns": len(response), + "SW_ns": np.sum(w_des if weight_type == "des" else w_iv), + } + ) + for key in ("R_S11", "R_S12", "R_S21", "R_S22"): + header[key] = 0.0 + with fits.HDUList( + [fits.PrimaryHDU(header=header), fits.BinTableHDU.from_columns(columns)] + ) as hdul: + hdul.writeto(path) + return {"des": w_des, "iv": w_iv}, shear > 0 + + +@pytest.mark.parametrize("weight_type", ["des", "iv"]) +def test_recalibrated_weighted_shear_uses_weighted_response(tmp_path, weight_type): + """The weighted recalibrated patch shear must recover g=0.02 exactly. + + Patch P has responses 0.5 and 1.5 with opposite w_des/w_iv preferences. + Intrinsic +/- pairs in patches with opposite shears make the global + weighted mean ellipticity zero, so c=0. The patch's weighted uncalibrated + mean is exactly _w*g. Dividing by the matching response (des: 1.25, + iv: 0.75) recovers g; the unweighted response 1.0 gives 1.25*g or 0.75*g. + """ + path = tmp_path / "cat.fits" + weights, in_patch = _write_catalogue(path, weight_type) + w = weights[weight_type] + g1, _ = get_calibrate_e_from_cat(str(path), weight_type=weight_type) + g_hat = np.average(g1[in_patch], weights=w[in_patch]) + assert g_hat == pytest.approx(G_TRUE, rel=0, abs=1e-12), ( + f"weight_type={weight_type!r}: weighted patch shear {g_hat:.6f} " + f"!= truth {G_TRUE}; response and additive bias must share weights" + ) diff --git a/src/sp_validation/tests/test_response_diagnostics.py b/src/sp_validation/tests/test_response_diagnostics.py new file mode 100644 index 00000000..d80276c6 --- /dev/null +++ b/src/sp_validation/tests/test_response_diagnostics.py @@ -0,0 +1,60 @@ +"""Brute-force checks of the explicitly shear-only #23 stage-1 diagnostic.""" + +import numpy as np + +from sp_validation.response_diagnostics import pair_products + + +def test_all_pair_tensor_products_against_brute_force(): + ra = np.array([0.0, 0.03, 0.07, 0.13, 0.21, 0.32]) + dec = np.zeros(6) + w = np.array([1.0, 4.0, 2.0, 3.0, 1.0, 2.0]) + r = np.array( + [ + np.linspace(0.6, 1.1, 6), + np.linspace(-0.1, 0.2, 6), + np.linspace(0.2, -0.05, 6), + np.linspace(0.8, 1.2, 6), + ] + ) + result = pair_products( + ra, dec, w, r.reshape(2, 2, 6), min_sep=1, max_sep=30, nbins=3 + ) + sums = np.zeros((3, 4, 4)) + den = np.zeros(3) + for i in range(6): + for j in range(6): + if i == j: + continue + theta = abs(ra[i] - ra[j]) * 60 + if not 1 <= theta < 30: + continue + k = int(np.log(theta) / (np.log(30) / 3)) + sums[k] += w[i] * w[j] * np.outer(r[:, i], r[:, j]) + den[k] += w[i] * w[j] + np.testing.assert_allclose( + result["products"], sums / den[:, None, None], rtol=2e-7, atol=1e-9 + ) + # Spatial variation is measurable; the global product need not suffice. + assert ( + np.max(abs(result["products"][:, 0, 0] / result["global_products"][0, 0] - 1)) + > 0.02 + ) + + +def test_constant_full_matrix_has_exact_pair_product(): + matrix = np.array([[0.8, 0.1], [-0.02, 0.9]]) + rg = np.repeat(matrix[:, :, None], 6, axis=2) + result = pair_products( + np.arange(6) * 0.05, + np.zeros(6), + np.arange(6) + 1.0, + rg, + min_sep=1, + max_sep=30, + nbins=3, + ) + expected = np.outer(matrix.ravel(), matrix.ravel()) + np.testing.assert_allclose( + result["products"], np.broadcast_to(expected, (3, 4, 4)), rtol=2e-7, atol=1e-9 + ) diff --git a/src/sp_validation/tests/test_response_weights.py b/src/sp_validation/tests/test_response_weights.py new file mode 100644 index 00000000..25ec4256 --- /dev/null +++ b/src/sp_validation/tests/test_response_weights.py @@ -0,0 +1,92 @@ +"""Branch IV and refitted-table regression tests.""" + +import numpy as np +import pytest + +from sp_validation import calibration, response +from sp_validation.tests.test_consistent_response import catalogue + + +def test_extraction_carries_available_branch_errors_but_allows_historical_des(): + data, _, _ = catalogue() + carried = response.branch_error_columns(data) + assert len(carried) == 8 + for key, value in carried.items(): + np.testing.assert_array_equal(value, data[key]) + historical = data[[k for k in data.dtype.names if k not in carried]] + assert response.branch_error_columns(historical) == {} + # Requiring these columns for DES would break existing merged inputs. + calibration.metacal( + historical, + np.ones(len(historical), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + + +def test_iv_branch_variances_and_normalization(): + data, _, _ = catalogue() + data["NGMIX_G1_ERR_1P"] = np.linspace(0.1, 0.8, len(data)) + data["NGMIX_G2_ERR_1M"] = np.linspace(0.8, 0.1, len(data)) + gm = calibration.metacal( + data, np.ones(len(data), bool), weight_scheme="iv", size_corr_ell=False + ) + for b, suffix in (("p1", "1P"), ("m1", "1M")): + expected = 1 / ( + 2 * 0.34**2 + + data[f"NGMIX_G1_ERR_{suffix}"] ** 2 + + data[f"NGMIX_G2_ERR_{suffix}"] ** 2 + ) + np.testing.assert_allclose(gm.branch_weights[b], expected) + expected = ( + np.average(gm.branches["ns"], weights=gm.branch_weights["p1"], axis=1) + - np.average(gm.branches["ns"], weights=gm.branch_weights["m1"], axis=1) + ) / 0.02 + np.testing.assert_allclose(gm.R_selection[:, 0], expected) + assert abs(expected[0]) > 0.01 + + +def test_refit_w_jackknife_matches_retained_training_sample(): + data, _, _ = catalogue() + # Unequal group noise at the same SNR changes fitted W on deletion. + for b in ("NOSHEAR", "1P", "1M", "2P", "2M"): + data[f"NGMIX_G1_{b}"][:20] += 0.3 + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + groups = np.repeat(np.arange(4), 20) + labels, jk, cov = gm.response_jackknife(groups, refit_w=True) + for j, label in enumerate(labels): + keep = groups != label + retained = data[keep] + other = calibration.metacal( + retained, + np.ones(len(retained), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + np.testing.assert_allclose(jk["c_g"][j], other.c_g, atol=1e-13) + np.testing.assert_allclose(jk["total"][j], other.R, atol=1e-13) + fixed = gm.response_jackknife(groups)[1] + assert np.max(abs(jk["c_g"] - fixed["c_g"])) > 1e-3 + vectors = np.concatenate([x.reshape(len(labels), -1) for x in jk.values()], axis=1) + delta = vectors - vectors.mean(axis=0) + np.testing.assert_allclose(cov, 0.75 * delta.T @ delta) + weights = gm.refit_weights(np.ones(len(data), bool)) + for b in response.BRANCHES: + np.testing.assert_allclose(weights[b], gm.branch_weights[b]) + + +def test_iv_missing_branch_errors_requires_reextraction(): + data, _, _ = catalogue() + data = data[[k for k in data.dtype.names if k != "NGMIX_G1_ERR_1P"]] + with pytest.raises(ValueError, match="regenerate.*comprehensive"): + calibration.metacal( + data, np.ones(len(data), bool), weight_scheme="iv", size_corr_ell=False + ) diff --git a/src/sp_validation/tests/test_sparse_des.py b/src/sp_validation/tests/test_sparse_des.py new file mode 100644 index 00000000..c55f8e11 --- /dev/null +++ b/src/sp_validation/tests/test_sparse_des.py @@ -0,0 +1,126 @@ +"""Minimum-count DES moments and identical full/deleted training rules.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import calibration, response +from sp_validation.tests.test_consistent_response import catalogue + + +def sparse_fixture(): + # Two nearly round objects would otherwise get W=640000. + snr = np.r_[np.repeat(20.0, 40), [100.0, 100.0]] + size = np.ones(len(snr)) + e = np.full((2, len(snr)), 0.2) + e[:, -2:] = 0.001 + r = np.full(len(snr), 0.8) + edges = np.array([10.0, 50.0, 500.0]), np.array([0.5, 2.0, 3.0]) + return snr, size, e, r, *edges + + +def test_sparse_cell_pools_moments_not_weights(): + args = sparse_fixture() + raw = response.fit_w(*args) + pooled = response.fit_w(*args, min_count=3) + assert raw[2][1, 0] > 600000 + expected = 0.8**2 / np.mean(args[2] ** 2) + np.testing.assert_allclose(pooled[2][1, 0], expected) + assert pooled[2][1, 0] < 17 + # Dense cell is bitwise unchanged; training counts stay unmerged. + assert pooled[2][0, 0] == raw[2][0, 0] + np.testing.assert_array_equal(pooled[3], raw[3]) + # A newly occupied artificial-branch cell uses the same pooled table. + np.testing.assert_allclose(response.evaluate_w(pooled, [100.0], [2.5]), expected) + + +def test_sparse_deletion_is_deterministic_and_order_independent(): + snr, size, e, r, es, er = sparse_fixture() + for deleted in (0, 40, 41): + keep = np.arange(len(snr)) != deleted + args = (snr[keep], size[keep], e[:, keep], r[keep], es, er) + first = response.fit_w(*args, min_count=30) + second = response.fit_w(*args, min_count=30) + reverse = response.fit_w( + snr[keep][::-1], + size[keep][::-1], + e[:, keep][:, ::-1], + r[keep][::-1], + es, + er, + min_count=30, + ) + np.testing.assert_array_equal(first[2], second[2]) + np.testing.assert_allclose(first[2], reverse[2], rtol=1e-14) + assert first[2][1, 0] < 17 + + +def test_fully_populated_table_is_bitwise_unchanged(): + snr = np.repeat([20.0, 20.0, 100.0, 100.0], 30) + size = np.repeat([1.0, 2.5, 1.0, 2.5], 30) + e = np.array([np.linspace(0.1, 0.4, len(snr))] * 2) + args = ( + snr, + size, + e, + np.full(len(snr), 0.8), + np.array([10.0, 50.0, 500.0]), + np.array([0.5, 2.0, 3.0]), + ) + for actual, expected in zip( + response.fit_w(*args, min_count=30), response.fit_w(*args), strict=True + ): + np.testing.assert_array_equal(actual, expected) + + +def test_neighbourhood_grows_until_threshold_and_handles_small_total(): + snr = np.r_[15.0, np.repeat(150.0, 10), np.repeat(1500.0, 40)] + size = np.ones(len(snr)) + e = np.array([np.r_[0.001, np.repeat(0.1, 10), np.repeat(0.3, 40)]] * 2) + args = ( + snr, + size, + e, + np.ones(len(snr)), + np.array([10.0, 100.0, 1000.0, 10000.0]), + np.array([0.5, 2.0]), + ) + near = response.fit_w(*args, min_count=10) + far = response.fit_w(*args, min_count=30) + np.testing.assert_allclose(near[2][0, 0], 1 / np.mean(e[:, :11] ** 2)) + np.testing.assert_allclose(far[2][0, 0], 1 / np.mean(e**2)) + small = response.fit_w(*args, min_count=100) + np.testing.assert_allclose(small[2], 1 / np.mean(e**2)) + with pytest.raises(ValueError, match="nonnegative"): + response.fit_w(*args, min_count=-1) + + +def test_production_refit_uses_same_minimum_count_rule(tmp_path): + d, _, _ = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + des_min_count=30, + ) + keep = np.arange(len(d)) % 4 != 0 + refitted = gm.refit_weights(keep) + rebuilt = calibration.metacal( + d[keep], + np.ones(keep.sum(), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + des_min_count=30, + ) + for b in response.BRANCHES: + np.testing.assert_allclose(refitted[b][keep], rebuilt.branch_weights[b]) + path = tmp_path / "response.fits" + fits.HDUList([fits.PrimaryHDU()]).writeto(path) + calibration.save_response_product(path, gm) + with fits.open(path) as hdus: + assert hdus[0].header["DESNMIN"] == 30 + assert "pooled moments" in hdus[0].header["DESMERGE"] + np.testing.assert_array_equal(hdus["W_COUNT"].data, gm.W[3]) From 091f2889f1d105b0136568125cb84bac385e0e79 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 26 Oct 2026 19:51:23 +0100 Subject: [PATCH 26/36] Define weighted response, additive bias and simulation calibration --- docs/source/response_calibration.md | 79 +++++++++++++++++++++++++++++ docs/source/using_the_catalogues.md | 23 +++++---- 2 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 docs/source/response_calibration.md diff --git a/docs/source/response_calibration.md b/docs/source/response_calibration.md new file mode 100644 index 00000000..657aeb71 --- /dev/null +++ b/docs/source/response_calibration.md @@ -0,0 +1,79 @@ +# Response, weights and additive bias + +The applied estimator follows Martin Kilbinger's **Separate** definition on slide S1 (6 October 2026, PDF p14, [shared deck](https://docs.google.com/presentation/d/1BdPlRCf742AcpfJVrAe_c3-OOp1ARiwuU9RMNflb0U8/edit)). +DES weights follow **case d** in `/n23data1/mkilbing/imsim_wdes_response/wdes_response.py`, functions `w_table`, `eval_table`, `responses`. +Neither split nor joint response on aligned metacal rows includes detection response. + +For branch b, define A_b(x) = sum(S_b w_b x) / sum(S_b w_b). +Every branch has its own selection and denominator. +The artificial metacal step h defaults to 0.01; it is distinct from the manifest's externally injected simulation shear. + +| Quantity | Definition | +| --- | --- | +| W per cell | mean(trace R_g/2)^2 / mean((e1²+e2²)/2), unweighted raw no-shear moments | +| DES weights | Fit W once on S_ns; evaluate that same table at each branch's own SNR and T/Tpsf | +| IV weights | 1/(2 sigma_eps² + G1_ERR² + G2_ERR²), using each branch's own measured errors | +| R_shear | A_ns((e_j+ − e_j−)/(2h)), all four elements | +| R_sel | (A_j+(e_ns) − A_j−(e_ns))/(2h); includes cuts, weight changes and cell migration | +| R_total / R_joint | Apply R_shear + R_sel; save joint = (A_j+(e_j+) − A_j−(e_j−))/(2h) as a diagnostic | +| c_e / c_g | Raw c_e = A_ns(e); calibrated c_g = inverse(R_total) c_e | +| Applied shear | inverse(R_total) e − c_g; omit c_g subtraction for constant-shear simulations | + +## Configuration and compatibility + +```yaml +metacal: + weight_scheme: des + w_col: w_des + des_num_bins: 20 + des_min_count: 3 + additive_correction: true # false for constant-shear simulations +``` + +`metacal_from_config(data, preselection, config['metacal'])` is the common entry point. +A mismatched `w_col` raises an error. +Cut products declare `WGHTSCH`, `RAPPLIED`, `MCSTEP`, `WPOLICY` and `IVPOLICY`. +Consumers must use the matching weight and `R: 1`; the catalogue already received the full matrix. +The configured catalogue reader and image-simulation reader refuse a second R or a different numerator weight on these products. +External catalogues without these metadata keep their existing interpretation. + +At the Python API, `global_R_weight='w'` is deprecated and maps to `weight_scheme='iv'`, including squared G_ERR and weighted selection means. +Versioned release configurations without `weight_scheme` explicitly use the legacy response/c and unsquared-IV policy. +Extraction carries all ten branch G_ERR columns and writes corrected `w_iv` using the public variance helper; legacy calibration reconstructs the historical IV internally. +Modern IV requires sheared-branch errors: regenerate incomplete inputs from the tile-final catalogues and retain those columns in the merge allow-list. +The optional direct-calibration tail of `extract_info.py` refuses because it lacks a shared weight/selection configuration; use the comprehensive product and calibration driver. + +DES cells are right-closed, including internal boundaries. +Finite coordinates outside the table range use the nearest edge cell; nonfinite coordinates get zero weight. +Below `des_min_count`, training moments are pooled over the smallest clipped concentric square reaching that count (or the entire table if necessary). +Counts remain the original, unpooled counts; unpooled empty/invalid cells get zero weight. +A row selected only on a sheared branch still receives a fixed-table weight. + +## Spatial uncertainties and simulation calibration + +`save_response_product` adds W_DES, W_COUNT, W_EDGES, R_JK and R_COV extensions when spatial groups are supplied. +R_JK deletes each shared TILE_ID from every branch and recomputes the selected population and its denominator. +R_COV uses (K−1)/K times the sum of outer products of centered delete-one estimates. +Its ordering is row-major R_shear, R_sel, R_total, R_joint, c_e, c_g (20 elements). +For a self-fitted DES table, the writer refits W on each retained no-shear sample, evaluates it on all five artificial branches and stores the calibrated deleted means in `R_JK.c_g1,c_g2`. +`response_jackknife(groups, refit_w=False)` supplies the conditional fixed-W comparison. +The returned c_err is a weighted error of the raw mean (legacy mode keeps historical scatter); correlated response/c uncertainty belongs in R_COV. + +Image simulations use a named, independent DES data-table product, declared by `image_sim_config.calibration_config` and the image-simulation workflow configuration. +The product contains W, counts and bin edges, with its name recorded in the calibrated catalogue metadata. +Self-fitted weights correlate with the measured shapes: the deterministic test in `test_des_table_independence.py` gives m = −1/N_cell for self-fitting and zero for independent training. +An external table stays fixed during spatial deletions; its training uncertainty is not included. +The production `ImageSimMBias` estimator uses full selected populations with separate branch weights and shared-tile response-recalibrating jackknife covariance for (m1,c1,m2,c2). +Modern products reject pair matching; symmetric pair weights omit a numerator contribution retained in the response and are diagnostic only. +Residual m remains a separate result, not an extra correction applied silently to catalogue e. + +## Diagnostics and reproduction + +`response_diagnostics.plot_response` plots per-object R_g, spatial delete-one R_sel/total/joint, split−joint differences and tile-mean R_g maps. +R_sel is an ensemble derivative, never a per-object response. +`pair_products` measures all 16 angle-binned, pair-weighted products of R_g elements. +`scalar_pair_response` measures _ww/_w²−1 with r=trace(R_g)/2 and shared spatial deletions. +These shear-only diagnostics omit selection/weight derivatives and cannot certify total two-point response adequacy. + +See `scripts/validation/response/README.md` for unblinded case-d reproduction, estimator comparison, independent-table residual m and occupancy-reweighted m. +No generated measurements or plots are stored in git. diff --git a/docs/source/using_the_catalogues.md b/docs/source/using_the_catalogues.md index a3817a3c..b6bbfc6b 100644 --- a/docs/source/using_the_catalogues.md +++ b/docs/source/using_the_catalogues.md @@ -134,18 +134,21 @@ _ = plt.ylabel(r'$\xi_\pm(\theta)$') ## 2. Extended catalogue -This catalogue must be **calibrated before use**. It carries the information -needed for calibration with the method of *metacalibration* -([arXiv:1702.02600](https://arxiv.org/abs/1702.02600)). Cuts may be applied -*before* calibration. +This catalogue must be **calibrated before use**. +A cut catalogue containing only per-object R_g and one global selection response does **not** contain enough information to recalibrate a changed sample or weight scheme. +Use the parent five-branch comprehensive catalogue and `calibration.metacal_from_config` with matching cuts and `weight_scheme: des` (or `iv`). +See [Response definitions](response_calibration.md). +The historical recipe below is retained only to explain the released columns; its reuse of global R_s after new cuts is not a valid new calibration. ```{tip} The `sp_validation` library performs every step below for you: {func}`sp_validation.calibration.get_calibrated_m_c` calibrates an in-memory catalogue for multiplicative and additive bias, and {func}`sp_validation.calibration.get_calibrate_e_from_cat` does the same from a -catalogue on disk. The worked example here is kept for understanding what those -functions compute. +catalogue on disk when its stored weight identity matches. +Unknown/mismatched selection-response metadata is refused, as are changed row counts or weight sums. +Even a historically zero selection response does not establish that it stays zero after new cuts. +Neither function reconstructs missing five-branch inputs after new cuts. ``` ### Open the file @@ -207,16 +210,16 @@ ensemble $R_g$ is computed *after* the sub-sample selection: R_g = np.empty((2, 2)) for idx in (0, 1): for jdx in (0, 1): - R_g[idx, jdx] = np.mean(data_ext_sub[f'R_g{idx+1}{jdx+1}']) + R_g[idx, jdx] = np.average(data_ext_sub[f'R_g{idx+1}{jdx+1}'], weights=data_ext_sub['w']) print('Shear response matrix R_g =') print(np.matrix(R_g)) ``` The selection response was pre-computed for the *entire* sample and cannot be -obtained per galaxy, so this catalogue version reuses the global -$R_\mathrm{s}$ for sub-samples too. Since $|R_\mathrm{s}| < |R_g|$ the resulting -error is small. +obtained per galaxy. +Do not reuse it for a new selection or weight identity: its small magnitude does not bound the error introduced by doing so. +The following arithmetic is valid only for the original, unchanged population with matching calibration weights. ```python R_s = np.empty((2, 2)) From db6a2480873f5c58610866e1236e2ee88814eddb Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 26 Oct 2026 19:51:33 +0100 Subject: [PATCH 27/36] Consolidate reproducible response validation commands --- scripts/validation/response/README.md | 50 +++ scripts/validation/response/inputs.py | 98 ++++ scripts/validation/response/m_split.py | 383 ++++++++++++++++ .../response/occupancy_residual_m.py | 420 ++++++++++++++++++ .../response/plot_response_m_estimators.py | 68 +++ .../response/sim_vs_data_response.py | 263 +++++++++++ .../validation/response/summarize_response.py | 43 ++ .../summarize_sim_vs_data_response.py | 47 ++ .../validation/response/summarize_v2_table.py | 38 ++ scripts/validation/response/v2_data_table.py | 93 ++++ .../validation/response/validate_response.py | 233 ++++++++++ 11 files changed, 1736 insertions(+) create mode 100644 scripts/validation/response/README.md create mode 100644 scripts/validation/response/inputs.py create mode 100644 scripts/validation/response/m_split.py create mode 100644 scripts/validation/response/occupancy_residual_m.py create mode 100644 scripts/validation/response/plot_response_m_estimators.py create mode 100644 scripts/validation/response/sim_vs_data_response.py create mode 100644 scripts/validation/response/summarize_response.py create mode 100644 scripts/validation/response/summarize_sim_vs_data_response.py create mode 100644 scripts/validation/response/summarize_v2_table.py create mode 100644 scripts/validation/response/v2_data_table.py create mode 100644 scripts/validation/response/validate_response.py diff --git a/scripts/validation/response/README.md b/scripts/validation/response/README.md new file mode 100644 index 00000000..e787cd6b --- /dev/null +++ b/scripts/validation/response/README.md @@ -0,0 +1,50 @@ +# Unblinded response validation + +Run from the repository root inside the container on an allocated compute node. +Prefix each command with `srun --jobid= -n 1 -c 8 --mem=45G spv-container exec` and set `PYTHONPATH=src`. +Choose an output directory outside git, e.g. `OUT="$PWD/response-results"`. +The commands below use the shared simulation and reference directories and the archived independent DES table by default; no personal scratch inputs are required. + +## Case-d reproduction and estimator comparison + +```bash +for g in 1 2 3 4; do + python scripts/validation/response/validate_response.py --grid "$g" --output "$OUT/martin/grid_$g" +done +python scripts/validation/response/summarize_response.py "$OUT/martin" "$OUT/martin" +python scripts/validation/response/plot_response_m_estimators.py "$OUT/martin" "$OUT/martin" +``` + +The summary asserts agreement in R, W and paired m to 1e-14 on all 20 products, using Martin's unpooled cells (`N_min=0`). +The plot compares symmetric paired, matched branch-weighted and full-population estimators. + +## Independent-table production residual m + +```bash +for g in 1 2 3 4; do + python scripts/validation/response/v2_data_table.py --grid "$g" --output "$OUT/production" +done +python scripts/validation/response/summarize_v2_table.py "$OUT/production" +``` + +## Shear-only and selection-response split + +Measure full-matrix shear-only m and the selection-response contribution with identical numerators and shared-tile deletions, combining four grids with common total-m weights to preserve the split. + +```bash +python scripts/validation/response/m_split.py --output "$OUT/m_split" +``` + +## Simulation/data occupancy and response + +```bash +python scripts/validation/response/sim_vs_data_response.py --output "$OUT/occupancy" +python scripts/validation/response/summarize_sim_vs_data_response.py "$OUT/occupancy" +python scripts/validation/response/occupancy_residual_m.py --output "$OUT/occupancy" --production "$OUT/production" +``` + +Measurement scripts accept `--base` (Martin's comprehensive grids), `--reference` (his calibrated case-d products), `--product` and `--product-name` (independent DES data table). +Their `--help` shows site-specific defaults; the table defaults to the named `v2-d533a078` FITS product at `/n17data/cdaley/unions/validation/2026-10-26_response-425/products/des_weight_product-d533a078.fits`. +That directory's `INDEX.md` records the table's provenance and checksum; use `--product /path/to/des_weight_product-d533a078.fits` to override its location. +FITS, JSON, NPZ and PNG outputs all go under the supplied output directories; no archived review-round outputs are required. +The estimator plot optionally accepts `--production "$OUT/production"` after the production summary exists. diff --git a/scripts/validation/response/inputs.py b/scripts/validation/response/inputs.py new file mode 100644 index 00000000..770676f4 --- /dev/null +++ b/scripts/validation/response/inputs.py @@ -0,0 +1,98 @@ +"""Shared unblinded grid inputs, branch-error join and response helpers.""" + +from pathlib import Path + +import numpy as np +import yaml +from astropy.io import fits +from numpy.lib import recfunctions + +from sp_validation import calibration, catalog_builders, io, response + +BASE = Path( + "/automnt/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow" +) +REFERENCE = Path("/n23data1/mkilbing/imsim_wdes_response") +PRODUCT = dict( + name="v2-d533a078", + path=( + "/n17data/cdaley/unions/validation/2026-10-26_response-425/" + "products/des_weight_product-d533a078.fits" + ), +) +EXTERNAL = ("1p2z", "1m2z", "1z2p", "1z2m") + + +def add_arguments(parser): + parser.add_argument("--base", type=Path, default=BASE) + parser.add_argument("--reference", type=Path, default=REFERENCE) + parser.add_argument("--product", type=Path, default=Path(PRODUCT["path"])) + parser.add_argument("--product-name", default=PRODUCT["name"]) + + +def configure(args): + global BASE, REFERENCE, PRODUCT + BASE, REFERENCE = args.base, args.reference + PRODUCT = dict(name=args.product_name, path=str(args.product)) + + +def load(grid, name, scheme, exact=True, min_count=None): + directory = BASE / f"grid_{grid}/{name}_grid_{grid}" + with open(directory / "config_mask.yaml") as f: + cfg = yaml.safe_load(f) + data = io.read_catalogue(directory / "shape_catalog_comprehensive_ngmix.fits") + if scheme == "iv" and exact: + # Tile finals supply branch errors absent from the comprehensive inputs. + missing = [ + f"NGMIX_G{i}_ERR_{s}" for s in ("1P", "1M", "2P", "2M") for i in (1, 2) + ] + errors = {k: np.full(len(data), np.nan) for k in missing} + tile = np.char.strip(np.asarray(data["TILE_ID"]).astype(str)) + for label in np.unique(tile): + idx = np.flatnonzero(tile == label) + disk_label = f"{float(label):.3f}" + paths = list( + (directory / "product/tiles").glob(f"*/{disk_label}/final_cat-*.fits") + ) + if not paths: + raise FileNotFoundError(f"{directory}: tile {label!r}") + with fits.open(paths[0]) as hdus: + raw = hdus[1].data + order = np.argsort(raw["NUMBER"]) + pos = order[np.searchsorted(raw["NUMBER"][order], data["NUMBER"][idx])] + np.testing.assert_array_equal(raw["NUMBER"][pos], data["NUMBER"][idx]) + for k in ("NGMIX_G1_NOSHEAR", "NGMIX_G2_ERR_NOSHEAR"): + np.testing.assert_allclose( + raw[k][pos], data[k][idx], equal_nan=True + ) + for k in missing: + errors[k][idx] = raw[k][pos] + data = recfunctions.append_fields( + np.asarray(data), list(errors), list(errors.values()), usemask=False + ) + masks, _ = catalog_builders.get_masks_from_config(cfg, data, verbose=False) + pre = catalog_builders.Mask.from_list(masks, label="combined", verbose=False)._mask + cm = cfg["metacal"].copy() + cm.pop("global_R_weight", None) + cm.update(weight_scheme=scheme, w_col=f"w_{scheme}", des_num_bins=20) + if min_count is not None: + cm["des_min_count"] = min_count + gm = calibration.metacal_from_config(data, pre, cm) + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + return gm, tiles + + +def covariance(x): + delta = x - x.mean(axis=0) + return (len(x) - 1) / len(x) * delta.T @ delta + + +def table_weights(gm, table): + return { + b: response.evaluate_w( + table, + getattr(gm, b)["flux"] / getattr(gm, b)["flux_err"], + getattr(gm, b)["T"] / getattr(gm, b)["Tpsf"], + ) + for b in response.BRANCHES + } diff --git a/scripts/validation/response/m_split.py b/scripts/validation/response/m_split.py new file mode 100644 index 00000000..2576166d --- /dev/null +++ b/scripts/validation/response/m_split.py @@ -0,0 +1,383 @@ +"""Measure the full-matrix shear/selection calibration split on unblinded grids. + +Run inside the sp_validation container on a Slurm compute node. All writes go +under --output. Inputs follow the sibling response validation scripts. +The numerator is held identical between the shear-only and total calibration. +""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + +import matplotlib +import numpy as np +import yaml + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +from inputs import EXTERNAL + +from sp_validation import calibration, catalog_builders, image_sim_config, io, response +from sp_validation.image_sims import ImageSimMBias + +CONFIGS = ("selection_des", "selection_unweighted", "no_selection_unweighted") +METRICS = ( + "R_shear", + "R_sel", + "m_shear_only", + "m", + "delta_m_sel", + "N_slope", + "scalar_m_shear_only", + "scalar_m", + "scalar_delta_m_sel", +) +ORDER = [f"{metric}_{i}" for metric in METRICS for i in (1, 2)] + + +def covariance(draws): + delta = draws - draws.mean(axis=0) + return (len(draws) - 1) / len(draws) * delta.T @ delta + + +def package(value, draws, labels): + cov = covariance(draws) + return dict( + value=np.asarray(value).tolist(), + error=np.sqrt(np.maximum(0, np.diag(cov))).tolist(), + covariance=cov.tolist(), + deleted=draws.tolist(), + labels=labels.tolist(), + order=ORDER, + ) + + +def estimator(config, cats, matrix): + """Use ImageSimMBias.run, supplying the same in-memory modern catalogue fields + its FITS loader supplies; swap only the applied response for the split. + """ + est = ImageSimMBias({**config, "w_cols": ["none"], "pair_match": False}) + est.cats = {} + for name, cat in cats.items(): + r = cat["full"][matrix] + jr = cat["jk"][matrix] + calibrated = np.linalg.solve(r, cat["raw"]) + est.cats[name] = dict( + weight_scheme="des", + W_policy="fixed-external", + e1=calibrated[0], + e2=calibrated[1], + raw=cat["raw"], + w={"none": cat["weights"]}, + tiles=cat["tiles"], + jk_labels=cat["labels"], + jk_R=jr, + ) + return est.run(False) + + +def run_grid(grid, output, production=None): + cfg_m = yaml.safe_load( + (inputs.BASE / f"grid_{grid}/results/m_bias_config.yaml").read_text() + ) + all_cats = {key: {} for key in CONFIGS} + reference_labels = None + parent_configs = {} + # Only four externally sheared branches enter m; no-external-shear catalogue + # is not used by ImageSimMBias.run for the modern full-population estimator. + for name in EXTERNAL: + directory = inputs.BASE / f"grid_{grid}/{name}_grid_{grid}" + original = yaml.safe_load((directory / "config_mask.yaml").read_text()) + parent_configs[name] = original + data = io.read_catalogue(directory / "shape_catalog_comprehensive_ngmix.fits") + masks, _ = catalog_builders.get_masks_from_config(original, data, verbose=False) + pre = catalog_builders.Mask.from_list( + masks, label="combined", verbose=False + )._mask + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + for key in CONFIGS: + cfg = image_sim_config.calibration_config( + original, ["w_des"], source="external", product=inputs.PRODUCT + ) + if key == "no_selection_unweighted": + cm = cfg["metacal"] + cm.update( + gal_snr_min=-np.inf, + gal_snr_max=np.inf, + gal_rel_size_min=-np.inf, + gal_rel_size_max=np.inf, + ) + gm = calibration.metacal_from_config(data, pre, cfg["metacal"]) + if key != "selection_des": + # Retain selection and per-object responses; use unit weights. + gm.branch_weights = {b: np.ones(len(tiles)) for b in response.BRANCHES} + full = gm.response_estimates() + labels, jk, _ = gm.response_jackknife(tiles, refit_w=False) + if reference_labels is None: + reference_labels = labels + np.testing.assert_array_equal(labels, reference_labels) + idx = gm.mask_dict["ns"] + raw = np.array(gm.branches["ns"])[:, idx] + all_cats[key][name] = dict( + full=full, + jk=jk, + labels=labels, + raw=raw, + weights=gm.branch_weights["ns"][idx], + tiles=tiles[idx], + counts={b: len(gm.mask_dict[b]) for b in response.BRANCHES}, + parent_count=len(tiles), + nonfinite_coordinates={ + b: int( + np.sum( + ~np.isfinite( + getattr(gm, b)["flux"] / getattr(gm, b)["flux_err"] + ) + | ~np.isfinite(getattr(gm, b)["T"] / getattr(gm, b)["Tpsf"]) + ) + ) + for b in response.BRANCHES + }, + ) + print(grid, name, key, all_cats[key][name]["counts"], flush=True) + del gm + del data + result = dict(grid=grid, parent_configs=parent_configs, configurations={}) + for key, cats in all_cats.items(): + total = estimator(cfg_m, cats, "total") + shear = estimator(cfg_m, cats, "shear") + value = np.empty((len(METRICS), 2)) + draws = np.empty((len(reference_labels), len(METRICS), 2)) + for i, (p, m) in enumerate((EXTERNAL[:2], EXTERNAL[2:])): + for dst, field in ((0, "shear"), (1, "selection")): + value[dst, i] = ( + cats[p]["full"][field][i, i] + cats[m]["full"][field][i, i] + ) / 2 + draws[:, dst, i] = ( + cats[p]["jk"][field][:, i, i] + cats[m]["jk"][field][:, i, i] + ) / 2 + value[2, i], value[3, i] = shear[f"m{i + 1}"], total[f"m{i + 1}"] + draws[:, 2, i] = np.array(shear["mc_jackknife"])[:, 2 * i] + draws[:, 3, i] = np.array(total["mc_jackknife"])[:, 2 * i] + value[4, i] = value[3, i] - value[2, i] + draws[:, 4, i] = draws[:, 3, i] - draws[:, 2, i] + gin = cfg_m["shear_amplitude"] + value[5, i] = (cats[p]["full"]["c_e"][i] - cats[m]["full"]["c_e"][i]) / ( + 2 * gin + ) + draws[:, 5, i] = ( + cats[p]["jk"]["c_e"][:, i] - cats[m]["jk"]["c_e"][:, i] + ) / (2 * gin) + for dst, field in ((6, "shear"), (7, "total")): + value[dst, i] = ( + cats[p]["full"]["c_e"][i] / cats[p]["full"][field][i, i] + - cats[m]["full"]["c_e"][i] / cats[m]["full"][field][i, i] + ) / (2 * gin) - 1 + draws[:, dst, i] = ( + cats[p]["jk"]["c_e"][:, i] / cats[p]["jk"][field][:, i, i] + - cats[m]["jk"]["c_e"][:, i] / cats[m]["jk"][field][:, i, i] + ) / (2 * gin) - 1 + value[8, i] = value[7, i] - value[6, i] + draws[:, 8, i] = draws[:, 7, i] - draws[:, 6, i] + entry = package( + value.ravel(), draws.reshape(len(reference_labels), -1), reference_labels + ) + entry["production_estimator"] = total + entry["branch_responses"] = { + name: dict( + full={k: v.tolist() for k, v in c["full"].items()}, + deleted={k: v.tolist() for k, v in c["jk"].items()}, + counts=c["counts"], + parent_count=c["parent_count"], + nonfinite_coordinates=c["nonfinite_coordinates"], + ) + for name, c in cats.items() + } + result["configurations"][key] = entry + if production is not None: + reference = json.loads((production / f"production_{grid}.json").read_text()) + total = result["configurations"]["selection_des"]["production_estimator"] + for name in ("m1", "m2", "m1_err", "m2_err", "mc_covariance", "mc_jackknife"): + np.testing.assert_allclose(total[name], reference[name], rtol=0, atol=1e-12) + result["production_reproduction_max_abs"] = max( + abs(total[k] - reference[k]) for k in ("m1", "m2", "m1_err", "m2_err") + ) + (output / f"grid_{grid}.json").write_text(json.dumps(result, indent=2)) + return result + + +def combine(grids, key, coherent=False): + entries = [g["configurations"][key] for g in grids] + values = np.array([e["value"] for e in entries]) + covs = np.array([e["covariance"] for e in entries]) + variances = np.diagonal(covs, axis1=1, axis2=2) + if coherent: + variances = np.tile(variances[:, 6:8], (1, len(METRICS))) + # Exact-zero selection response has zero covariance: use equal grid weights. + w = np.divide(1, variances, out=np.zeros_like(variances), where=variances > 0) + w[:, w.sum(axis=0) == 0] = 1 + w /= w.sum(axis=0) + cov = np.sum(covs * w[:, :, None] * w[:, None, :], axis=0) + return dict( + value=np.sum(w * values, axis=0).tolist(), + error=np.sqrt(np.maximum(0, np.diag(cov))).tolist(), + covariance=cov.tolist(), + grid_weights=w.tolist(), + order=ORDER, + ) + + +def plot(results, output): + labels = [ + "No SNR/size cuts\nUnweighted", + "SNR/size cuts\nUnweighted", + "SNR/size cuts\nDES (v2 table)", + ] + keys = CONFIGS[::-1] + fig, axes = plt.subplots(1, 2, figsize=(11, 4.4), sharey=True) + for i, ax in enumerate(axes): + for offset, metric, color, marker, label in ( + (-0.20, 2, "#0072B2", "o", "Shear-response only"), + (0, 4, "#D55E00", "s", "Selection contribution"), + (0.20, 3, "#222222", "D", "Total"), + ): + vals = [ + results["combined_common_m_weights"][k]["value"][2 * metric + i] * 1000 + for k in keys + ] + errs = [ + results["combined_common_m_weights"][k]["error"][2 * metric + i] * 1000 + for k in keys + ] + ax.errorbar( + np.arange(3) + offset, + vals, + yerr=errs, + fmt=marker, + color=color, + capsize=3, + label=label, + ms=5, + ) + ax.axhline(0, color=".6", lw=0.8) + ax.set_xticks(range(3), labels) + ax.set_title(f"Component {i + 1}") + ax.grid(axis="y", alpha=0.2) + axes[0].set_ylabel(r"Multiplicative bias $\times 10^3$") + axes[0].legend(fontsize=8) + fig.suptitle("Shear-only + selection contribution = total", fontsize=12) + fig.text( + 0.5, + 0.01, + "Four-grid means; common inverse-variance total-m weights preserve the split. Shared-tile jackknife errors.", + ha="center", + fontsize=8, + ) + fig.tight_layout(rect=(0, 0.04, 1, 0.94)) + fig.savefig(output / "m_split.png", dpi=180) + plt.close(fig) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(parser) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--production", + type=Path, + help="Check against existing per-grid production JSON", + ) + parser.add_argument( + "--reuse", action="store_true", help="Combine already measured grid files only" + ) + args = parser.parse_args() + inputs.configure(args) + args.output.mkdir(parents=True, exist_ok=True) + start_head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip() + grids = [ + json.loads((args.output / f"grid_{i}.json").read_text()) + if args.reuse + else run_grid(i, args.output, args.production) + for i in range(1, 5) + ] + results = dict( + metadata=dict( + checkout_head_start=start_head, + checkout_head_end=subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip(), + product=inputs.PRODUCT, + product_sha256=hashlib.sha256( + Path(inputs.PRODUCT["path"]).read_bytes() + ).hexdigest(), + assumption="Grids treated as independent, conditional on the named fixed external table; no external-table uncertainty or cross-grid tile covariance included.", + combined="Each reported quantity has its own inverse-jackknife-variance grid weights. These marginal means need not sum. combined_common_m_weights uses total-m weights for all quantities, preserving m = m_shear_only + delta_m_sel and used in the figure.", + response_summary="For component i, R_shear/R_sel are the average diagonal i,i across its positive and negative external branches. Every branch matrix and deletion is also saved; numerator calibration uses each branch matrix separately.", + decomposition="Exact production full-matrix counterfactual: same raw weighted population, numerator and deletions; replace R_total by R_shear. Scalar diagonal-only branchwise counterparts are also saved. There is no unique physical numerator split; changing selection also changes N, so delta_m_sel isolates only the calibration-denominator effect within a configuration.", + no_selection="Existing parent config dat cuts retained (FLAGS<=2, overlap=True, N_EPOCH>=2, mag 15..30, MCAL_TYPES_FAIL=0, no-shear PSF g1/g2 != -10); each artificial branch also requires NGMIX_FLAGS_branch=0. SNR/size bounds replaced by +/-infinity: NaN/infinite coordinates can still fail comparisons. Per-branch counts and nonfinite coordinate counts saved. Unit weights on all branches.", + ), + grids=grids, + combined={k: combine(grids, k) for k in CONFIGS}, + combined_common_m_weights={k: combine(grids, k, True) for k in CONFIGS}, + ) + # Keep covariance between configurations too: all use identical tile labels. + cross_order = [f"{key}:{name}" for key in CONFIGS for name in ORDER] + cross_covariances = [] + for grid in grids: + entries = grid["configurations"] + joint_draws = np.concatenate( + [np.asarray(entries[key]["deleted"]) for key in CONFIGS], axis=1 + ) + cross_cov = covariance(joint_draws) + grid["cross_configuration_covariance"] = dict( + order=cross_order, covariance=cross_cov.tolist() + ) + cross_covariances.append(cross_cov) + for key in CONFIGS: + entry = entries[key] + values = np.asarray(entry["value"]) + draws = np.asarray(entry["deleted"]) + np.testing.assert_allclose( + values[8:10], values[6:8] - values[4:6], atol=1e-14 + ) + np.testing.assert_allclose( + draws[:, 8:10], draws[:, 6:8] - draws[:, 4:6], atol=1e-14 + ) + # Confirm no hidden finite-coordinate or fit-flag rejection remains. + for branch in entries["no_selection_unweighted"]["branch_responses"].values(): + assert all(n == branch["parent_count"] for n in branch["counts"].values()) + assert not any(branch["nonfinite_coordinates"].values()) + for method in ("combined", "combined_common_m_weights"): + weights = np.concatenate( + [np.asarray(results[method][key]["grid_weights"]) for key in CONFIGS], + axis=1, + ) + cov = np.sum( + np.asarray(cross_covariances) * weights[:, :, None] * weights[:, None, :], + axis=0, + ) + results[f"{method}_cross_configuration_covariance"] = dict( + order=cross_order, covariance=cov.tolist() + ) + results["metadata"]["N_slope"] = ( + "Raw weighted external-shear slope: (A_plus(e_ns)-A_minus(e_ns))/(2*g_in). This is N/g in the question's notation." + ) + results["metadata"]["no_selection_verified"] = ( + "Every retained parent row passes all five artificial branch masks; nonfinite SNR/size counts are zero on all four grids. R_sel and its deletion covariance are exactly zero." + ) + v = np.array(results["combined"]["selection_des"]["value"])[6:8] + e = np.array(results["combined"]["selection_des"]["error"])[6:8] + print("Production combined m:", v, "+/-", e, flush=True) + (args.output / "results.json").write_text(json.dumps(results, indent=2)) + plot(results, args.output) + for key, entry in results["combined"].items(): + print(key, dict(zip(ORDER, zip(entry["value"], entry["error"]))), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/response/occupancy_residual_m.py b/scripts/validation/response/occupancy_residual_m.py new file mode 100644 index 00000000..57bb0091 --- /dev/null +++ b/scripts/validation/response/occupancy_residual_m.py @@ -0,0 +1,420 @@ +"""Full-population, fixed-v2-table m with fixed parent occupancy multipliers. + +Run on allocated nodes only. Use shared loading and response/JK kernels. +Anchor multipliers to no-external-shear detections and transfer by same-tile +sky position, never NUMBER (a detection sequence, not a cross-branch ID). +External-only detections extend the anchor catalogue on first appearance; +no population is restricted to matched objects. The anchor cell and multiplier +are then frozen across external/artificial branches and tile deletions. +SNR terciles instead use branch-specific SNR selection, including its R_sel. +""" + +import argparse +import json +from pathlib import Path +from unittest.mock import patch + +import matplotlib + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +import numpy as np +import yaml +from inputs import EXTERNAL, covariance, load, table_weights +from scipy.spatial import cKDTree + +from sp_validation import calibration, response + + +def read_grid(grid, name): + original = calibration.metacal_from_config + meta = {} + + def capture(data, pre, cfg): + # Retain reference coordinates even for detections excluded by preselection. + for k in ( + "RA", + "Dec", + "TILE_ID", + "NGMIX_FLUX_NOSHEAR", + "NGMIX_FLUX_ERR_NOSHEAR", + "NGMIX_T_NOSHEAR", + "NGMIX_T_PSF_RECONV_NOSHEAR", + ): + meta[k] = np.asarray(data[k]).copy() + meta["pre"] = np.flatnonzero(pre) + return original(data, pre, cfg) + + with patch.object(calibration, "metacal_from_config", capture): + gm, tiles = load(grid, name, "des", min_count=3) + return gm, tiles, meta + + +def xyz(ra, dec): + r, d = np.deg2rad(ra), np.deg2rad(dec) + return np.column_stack((np.cos(d) * np.cos(r), np.cos(d) * np.sin(r), np.sin(d))) + + +def index(snr, size, table): + return response.cell_index(snr, table[0]) * table[2].shape[1] + response.cell_index( + size, table[1] + ) + + +def meta_cells(meta, table): + with np.errstate(divide="ignore", invalid="ignore"): + snr = meta["NGMIX_FLUX_NOSHEAR"] / meta["NGMIX_FLUX_ERR_NOSHEAR"] + size = meta["NGMIX_T_NOSHEAR"] / meta["NGMIX_T_PSF_RECONV_NOSHEAR"] + return index(snr, size, table), np.isfinite(snr) & np.isfinite(size) + + +def measure(gm, tiles, w, snr_range=None): + masks, rg = gm.mask_dict, gm.R_shear + if snr_range is not None: + lo, hi = snr_range + masks = {} + for b, rows in gm.mask_dict.items(): + v = getattr(gm, b) + s = v["flux"][rows] / v["flux_err"][rows] + masks[b] = rows[(s > lo) & (s <= hi)] + pos = np.searchsorted(gm.mask_dict["ns"], masks["ns"]) + np.testing.assert_array_equal(gm.mask_dict["ns"][pos], masks["ns"]) + rg = gm.R_shear[:, :, pos] + point = response.responses(gm.branches, masks, w, rg, gm._step) + labels, jk, _ = response.jackknife(gm.branches, masks, w, rg, gm._step, tiles) + return point["c_g"], jk["c_g"], labels, {b: len(v) for b, v in masks.items()} + + +def mc(values, gin): + out = [] + for i, (p, m) in enumerate((("1p2z", "1m2z"), ("1z2p", "1z2m"))): + out.extend( + ( + (values[p][..., i] - values[m][..., i]) / (2 * gin) - 1, + (values[p][..., i] + values[m][..., i]) / 2, + ) + ) + return np.array(out).T + + +def package(value, deleted, labels): + cov = covariance(deleted) + return dict( + value=np.asarray(value).tolist(), + error=np.sqrt(np.diag(cov)).tolist(), + covariance=cov.tolist(), + deleted=deleted.tolist(), + labels=labels.tolist(), + ) + + +def combine(rows): + v = np.array([r["value"] for r in rows]) + e = np.array([r["error"] for r in rows]) + w = e**-2 + a = w / w.sum(0) + cov = sum( + np.array(r["covariance"]) * a[i, :, None] * a[i, None, :] + for i, r in enumerate(rows) + ) + return dict( + value=np.sum(a * v, 0).tolist(), + error=np.sqrt(np.diag(cov)).tolist(), + covariance=cov.tolist(), + weights=a.tolist(), + ) + + +def run(output, production): + path = output / "results.json" + previous = json.loads(path.read_text()) + table, _ = calibration.load_des_weight_product(inputs.PRODUCT) + es, er, wd, dc = table + pd = dc / dc.sum() + # Interpolate the W_COUNT CDF uniformly in log SNR inside occupied cells. + cdf = np.r_[0, np.cumsum(pd.sum(1))] + edges = np.exp(np.interp([0, 1 / 3, 2 / 3, 1], cdf, np.log(es))) + ranges = list(zip(edges[:-1], edges[1:])) + result = dict( + order=["m1", "c1", "m2", "c2"], + product=inputs.PRODUCT, + method="Fixed ns-external anchor-cell multipliers; same-tile sky transfer at production match radius, external-only detections extend anchors; no matching restriction. Fixed across artificial/external shears and JK deletions.", + tercile_method="Original (not occupancy-reweighted) populations; branch-specific SNR cuts with recalculated shear and selection responses. Data W_COUNT terciles, log-uniform interpolation within SNR bins.", + snr_tercile_edges=edges.tolist(), + grids={}, + ) + for grid in range(1, 5): + gm0, _, meta0 = read_grid(grid, "1z2z") + cell0, valid0 = meta_cells(meta0, table) + count = np.array(previous["grids"][str(grid)]["count"]) + factor = pd / (count / count.sum()) + ns = gm0.mask_dict["ns"] + np.testing.assert_array_equal( + np.bincount(cell0[meta0["pre"]][ns], minlength=400).reshape(wd.shape), count + ) + del gm0 + # Keep every initial raw detection as a positional anchor. Invalid raw + # coordinates in SNR/size get neutral occupancy, not a fabricated cell. + f0 = np.where(valid0, factor.ravel()[cell0], 1.0) + anchor = {} + tile0 = np.char.strip(meta0["TILE_ID"].astype(str)) + for t in np.unique(tile0): + use = tile0 == t + anchor[t] = ( + xyz(meta0["RA"][use], meta0["Dec"][use]), + f0[use], + np.zeros(use.sum(), bool), + ) + cfg = yaml.safe_load( + (inputs.BASE / f"grid_{grid}/results/m_bias_config.yaml").read_text() + ) + radius = 2 * np.sin(np.deg2rad(cfg["match_radius_deg"]) / 2) + means, deletes = {}, {} + counts, transfers = {}, {} + all_labels = None + for name in EXTERNAL: + gm, tiles, meta = read_grid(grid, name) + idx, valid = meta_cells(meta, table) + fallback = np.where(valid, factor.ravel()[idx], 1.0) + parent_factor = np.ones(len(meta["RA"])) + target_tile = np.char.strip(meta["TILE_ID"].astype(str)) + distance = np.zeros(len(parent_factor)) + fresh = np.zeros(len(parent_factor), bool) + external_origin = np.zeros(len(parent_factor), bool) + for t in np.unique(target_tile): + rows = np.flatnonzero(target_tile == t) + pos = xyz(meta["RA"][rows], meta["Dec"][rows]) + ap, af, external_anchor = anchor[t] + dist, nearest = cKDTree(ap).query(pos, workers=1) + matched = dist <= radius + parent_factor[rows] = np.where(matched, af[nearest], fallback[rows]) + distance[rows] = dist + fresh[rows] = ~matched + external_origin[rows] = ~matched | external_anchor[nearest] + if (~matched).any(): + anchor[t] = ( + np.concatenate((ap, pos[~matched])), + np.r_[af, fallback[rows[~matched]]], + np.r_[external_anchor, np.ones((~matched).sum(), bool)], + ) + density = parent_factor[meta["pre"]] + w = table_weights(gm, table) + policies = dict( + original=(w, None), + occupancy_matched=({b: v * density for b, v in w.items()}, None), + ) + # Sensitivity to the tiny population without any ns-external anchor. + neutral_density = np.where(external_origin[meta["pre"]], 1.0, density) + policies["external_only_neutral_check"] = ( + {b: v * neutral_density for b, v in w.items()}, + None, + ) + # Diagnostic only: quantify the consequence of re-evaluating density + # at each external branch's ns SNR instead of freezing object factors. + policies["external_ns_lookup_diagnostic"] = ( + {b: v * fallback[meta["pre"]] for b, v in w.items()}, + None, + ) + for i, limits in enumerate(ranges): + policies[f"tercile_{i + 1}"] = (w, limits) + for policy, (weights, limits) in policies.items(): + point, jk, labels, n = measure(gm, tiles, weights, limits) + if all_labels is None: + all_labels = labels + np.testing.assert_array_equal(labels, all_labels) + means.setdefault(policy, {})[name] = point + deletes.setdefault(policy, {})[name] = jk + counts.setdefault(policy, {})[name] = n + selected_raw = meta["pre"][gm.mask_dict["ns"]] + transfers[name] = dict( + new_anchors=int(fresh.sum()), + parent=len(fresh), + selected_new_anchors=int(fresh[selected_raw].sum()), + selected=len(selected_raw), + selected_external_origin_DES_weight_fraction=float( + w["ns"][gm.mask_dict["ns"]][external_origin[selected_raw]].sum() + / w["ns"][gm.mask_dict["ns"]].sum() + ), + selected_new_anchor_DES_weight_fraction=float( + w["ns"][gm.mask_dict["ns"]][fresh[selected_raw]].sum() + / w["ns"][gm.mask_dict["ns"]].sum() + ), + selected_match_99_percentile_arcsec=float( + np.rad2deg( + 2 * np.arcsin(np.quantile(distance[selected_raw], 0.99) / 2) + ) + * 3600 + ), + ) + print("loaded", grid, name, transfers[name], flush=True) + row = { + k: package( + mc(v, cfg["shear_amplitude"]), + mc(deletes[k], cfg["shear_amplitude"]), + all_labels, + ) + for k, v in means.items() + } + for k in ("occupancy_matched", "external_ns_lookup_diagnostic"): + row[k + "_minus_original"] = package( + np.array(row[k]["value"]) - row["original"]["value"], + np.array(row[k]["deleted"]) - row["original"]["deleted"], + all_labels, + ) + row["external_only_neutral_minus_matched"] = package( + np.array(row["external_only_neutral_check"]["value"]) + - row["occupancy_matched"]["value"], + np.array(row["external_only_neutral_check"]["deleted"]) + - row["occupancy_matched"]["deleted"], + all_labels, + ) + # Within-grid paired tercile contrast, including cross-tercile covariance. + row["high_minus_low_tercile"] = package( + np.array(row["tercile_3"]["value"]) - row["tercile_1"]["value"], + np.array(row["tercile_3"]["deleted"]) - row["tercile_1"]["deleted"], + all_labels, + ) + prod = json.loads((production / f"production_{grid}.json").read_text()) + p = prod["weights"]["w_des"] + np.testing.assert_array_equal(all_labels, prod["jackknife_tiles"]) + np.testing.assert_allclose( + row["original"]["value"], + [p[k] for k in result["order"]], + atol=1e-12, + rtol=0, + ) + np.testing.assert_allclose( + row["original"]["deleted"], p["mc_jackknife"], atol=1e-12, rtol=0 + ) + row["counts"] = counts + row["anchor_transfer"] = transfers + result["grids"][str(grid)] = row + print( + "GRID", + grid, + "matched", + row["occupancy_matched"]["value"], + "delta", + row["occupancy_matched_minus_original"]["value"], + flush=True, + ) + keys = list(means) + [ + "occupancy_matched_minus_original", + "external_ns_lookup_diagnostic_minus_original", + "external_only_neutral_minus_matched", + "high_minus_low_tercile", + ] + result["means"] = { + k: combine([r[k] for r in result["grids"].values()]) for k in keys + } + # Shift of original-weighted grid mean: same grid weights for both policies, + # with the covariance of paired deletions, not independent mean errors. + a = np.array(result["means"]["original"]["weights"]) + ds = [r["occupancy_matched_minus_original"] for r in result["grids"].values()] + value = np.sum(a * np.array([r["value"] for r in ds]), axis=0) + cov = sum( + np.array(r["covariance"]) * a[i, :, None] * a[i, None, :] + for i, r in enumerate(ds) + ) + result["means"]["shift_at_original_grid_weights"] = dict( + value=value.tolist(), + error=np.sqrt(np.diag(cov)).tolist(), + covariance=cov.tolist(), + ) + result["combination_note"] = ( + "Component-wise inverse variance, assuming independent grids. Paired differences combined using their own inverse variances; also save shift at common original grid weights. W and occupancy factors held fixed in JK; no data-table or density-estimation uncertainty included." + ) + previous["residual_m_occupancy"] = result + path.write_text(json.dumps(previous, indent=2)) + (output / "residual_m_occupancy.json").write_text(json.dumps(result, indent=2)) + fig, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True) + for j, component in enumerate((0, 2)): + ax = axes[0, j] + for k, label, offset in ( + ("original", "Original", -0.07), + ("occupancy_matched", "Data occupancy", 0.07), + ): + vals = [r[k]["value"][component] * 1000 for r in result["grids"].values()] + err = [r[k]["error"][component] * 1000 for r in result["grids"].values()] + ax.errorbar( + np.arange(1, 5) + offset, + vals, + yerr=err, + fmt="o", + capsize=3, + label=label, + ) + ax.axhline(0, color="gray", ls="--") + ax.set(xlabel="Grid", ylabel=f"m{j + 1} × 1000", xticks=range(1, 5)) + ax.legend() + ax = axes[1, j] + for grid, row in result["grids"].items(): + vals = [row[f"tercile_{i}"]["value"][component] * 1000 for i in (1, 2, 3)] + err = [row[f"tercile_{i}"]["error"][component] * 1000 for i in (1, 2, 3)] + ax.errorbar( + np.arange(1, 4) + (int(grid) - 2.5) * 0.035, + vals, + yerr=err, + fmt="o-", + capsize=2, + alpha=0.65, + label=f"Grid {grid}", + ) + vals = [ + result["means"][f"tercile_{i}"]["value"][component] * 1000 + for i in (1, 2, 3) + ] + err = [ + result["means"][f"tercile_{i}"]["error"][component] * 1000 + for i in (1, 2, 3) + ] + ax.errorbar( + range(1, 4), + vals, + yerr=err, + fmt="ks-", + capsize=4, + lw=2, + label="IV grid mean", + ) + ax.axhline(0, color="gray", ls="--") + ax.set( + xlabel="SNR tercile (data-occupancy boundaries)", + ylabel=f"m{j + 1} × 1000", + xticks=range(1, 4), + ) + ax.legend(fontsize=8) + fig.suptitle("Fixed v2 DES table; full populations; shared-tile jackknife") + fig.savefig(output / "residual_m_occupancy.png", dpi=180) + plt.close(fig) + for grid, row in result["grids"].items(): + print("\nRESULT GRID", grid) + for k in keys: + print( + k, + "value", + np.round(np.array(row[k]["value"]) * 1000, 4), + "err", + np.round(np.array(row[k]["error"]) * 1000, 4), + ) + print("\nMEANS") + for k, v in result["means"].items(): + print( + k, + "value", + np.round(np.array(v["value"]) * 1000, 4), + "err", + np.round(np.array(v["error"]) * 1000, 4), + ) + print("SNR tercile boundaries", edges) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(p) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--production", type=Path, required=True) + args = p.parse_args() + inputs.configure(args) + run(args.output, args.production) diff --git a/scripts/validation/response/plot_response_m_estimators.py b/scripts/validation/response/plot_response_m_estimators.py new file mode 100644 index 00000000..a1d73528 --- /dev/null +++ b/scripts/validation/response/plot_response_m_estimators.py @@ -0,0 +1,68 @@ +"""Compare paired, matched branch-weighted and full-population estimators.""" + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + + +def plot(root, output, production=None): + results = [ + json.loads((root / f"grid_{grid}/validation.json").read_text()) + for grid in range(1, 5) + ] + fig, axes = plt.subplots(1, 2, figsize=(11, 4.6), sharex=True) + x = np.arange(1, 5) + for component, ax in enumerate(axes, 1): + rows = [r["m"][f"m{component}"] for r in results] + for offset, key, label in ( + (-0.12, "new_paired", "Matched, symmetric pair weights (Martin)"), + (0, "intersection_branch_weighted", "Matched, separate branch weights"), + (0.12, "new_unpaired", "Full populations, separate branch weights"), + ): + error = ( + [r["unpaired_refit_W_tile_jk_error"] * 1000 for r in rows] + if key == "new_unpaired" + else None + ) + ax.errorbar( + x + offset, + [r[key] * 1000 for r in rows], + yerr=error, + fmt="o", + capsize=3, + label=label, + ) + if production is not None: + data = json.loads((production / "summary.json").read_text())["grids"] + idx = 2 * (component - 1) + ax.errorbar( + x + 0.24, + [data[str(i)]["value"][idx] * 1000 for i in x], + yerr=[data[str(i)]["error"][idx] * 1000 for i in x], + fmt="s", + capsize=3, + label="Full populations, independent v2 table", + ) + ax.axhline(0, color="black", lw=0.8) + ax.set(xticks=x, xlabel="Grid", ylabel=rf"$m_{component}$ ($10^{{-3}}$)") + ax.grid(axis="y", alpha=0.2) + axes[1].legend(frameon=False, fontsize=8) + fig.tight_layout() + output.mkdir(parents=True, exist_ok=True) + fig.savefig(output / "m_estimators.png", dpi=180, bbox_inches="tight") + plt.close(fig) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--production", type=Path) + args = parser.parse_args() + plot(args.root, args.output, args.production) diff --git a/scripts/validation/response/sim_vs_data_response.py b/scripts/validation/response/sim_vs_data_response.py new file mode 100644 index 00000000..76e7bb5f --- /dev/null +++ b/scripts/validation/response/sim_vs_data_response.py @@ -0,0 +1,263 @@ +"""Direct ns response and occupancy comparison; run on an allocated node only. + +Use the shared exact-error join, selection and independent-table evaluation. Density matching uses the ns cell's +p_data/p_sim as a fixed parent-object multiplier on every artificial branch: +it changes the population without adding an artificial density-weight derivative. +DES W and IV errors still vary with each artificial branch as in production. +""" + +import argparse +import json +from pathlib import Path +from unittest.mock import patch + +import matplotlib + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +import numpy as np +from inputs import load, table_weights + +from sp_validation import calibration, response + + +def pack(gm, weights): + r = response.responses(gm.branches, gm.mask_dict, weights, gm.R_shear, gm._step) + return { + k: dict( + matrix=r[k].tolist(), + diagonal=np.diag(r[k]).tolist(), + mean_diagonal=float(np.trace(r[k]) / 2), + ) + for k in ("shear", "selection", "total", "joint") + } + + +def ratios(results): + def r(a, b, kind): + return results[a][kind]["mean_diagonal"] / results[b][kind]["mean_diagonal"] + + return { + f"{kind}_{des}_over_{base}": r(des, base, kind) + for kind in ("shear", "total") + for des in ("des_v2", "des_self") + for base in ("unweighted", "iv_squared") + } + + +def run(output): + output.mkdir(parents=True, exist_ok=True) + table, nmin = calibration.load_des_weight_product(inputs.PRODUCT) + es, er, wd, data_count = table + pd = data_count / data_count.sum() + result = dict( + product=inputs.PRODUCT, + data_count=data_count.tolist(), + edges_snr=es.tolist(), + edges_size=er.tolist(), + density_matching="Fixed ns-cell p_data/p_sim on parent objects across all artificial branches; DES table fixed, not refitted after matching.", + grids={}, + ) + maps, cells, fractions = [], [], [] + for grid in range(1, 5): + # Capture legacy unsquared weights on the same joined input arrays. + old = {} + original = calibration.metacal.get_variance_ivweights + names = dict(NOSHEAR="ns", **{"1P": "p1", "1M": "m1", "2P": "p2", "2M": "m2"}) + + def capture(data, sigma_eps, mask=None, legacy=False, branch="NOSHEAR"): + old[names[branch]] = original(data, sigma_eps, mask, True, branch)[2] + return original(data, sigma_eps, mask, legacy, branch) + + with patch.object( + calibration.metacal, "get_variance_ivweights", staticmethod(capture) + ): + gm, tiles = load(grid, "1z2z", "iv", exact=True) + own, _ = load(grid, "1z2z", "des", min_count=3) + for b in response.BRANCHES: + np.testing.assert_array_equal(gm.mask_dict[b], own.mask_dict[b]) + np.testing.assert_allclose(gm.R_shear, own.R_shear) + np.testing.assert_array_equal(es, own.W[0]) + np.testing.assert_array_equal(er, own.W[1]) + policies = dict( + unweighted={b: np.ones(len(gm.ns["g1"])) for b in response.BRANCHES}, + iv_squared=gm.branch_weights, + iv_unsquared=old, + des_v2=table_weights(gm, table), + des_self=table_weights(gm, own.W), + ) + ns = gm.mask_dict["ns"] + snr = gm.ns["flux"] / gm.ns["flux_err"] + size = gm.ns["T"] / gm.ns["Tpsf"] + index = response.cell_index(snr, es) * len(wd[0]) + response.cell_index( + size, er + ) + shape = wd.shape + count = np.bincount(index[ns], minlength=wd.size).reshape(shape) + assert np.all(count > 0), "Data occupancy cannot be matched in empty sim cells" + ps = count / count.sum() + factor = pd / ps + density = factor.ravel()[index] + before = {k: pack(gm, w) for k, w in policies.items()} + after = { + k: pack(gm, {b: w[b] * density for b in response.BRANCHES}) + for k, w in policies.items() + } + rg = (gm.R_shear[0, 0] + gm.R_shear[1, 1]) / 2 + rsum = np.bincount(index[ns], weights=rg, minlength=wd.size).reshape(shape) + rcell = rsum / count + outside = ( + (snr[ns] < es[0]) + | (snr[ns] > es[-1]) + | (size[ns] < er[0]) + | (size[ns] > er[-1]) + ) + assert not outside.any() + # Check exact occupancy match and direct cell-sum shear identity. + matched = np.bincount( + index[ns], weights=density[ns], minlength=wd.size + ).reshape(shape) + np.testing.assert_allclose(matched / matched.sum(), pd, atol=1e-13) + np.testing.assert_allclose( + np.sum(pd * rcell), after["unweighted"]["shear"]["mean_diagonal"] + ) + summaries = dict( + median_snr=float(np.median(snr[ns])), + median_size=float(np.median(size[ns])), + fraction_snr_below_30=float(np.mean(snr[ns] < 30)), + occupancy_total_variation=float(np.abs(ps - pd).sum() / 2), + density_factor_min=float(factor.min()), + density_factor_max=float(factor.max()), + density_effective_n=float( + density[ns].sum() ** 2 / np.sum(density[ns] ** 2) + ), + per_cell_R_min=float(rcell.min()), + per_cell_R_max=float(rcell.max()), + ) + result["grids"][str(grid)] = dict( + selected=len(ns), + tiles=len(np.unique(tiles)), + sigma_eps=gm._sigma_eps, + responses=before, + ratios=ratios(before), + occupancy_matched_responses=after, + occupancy_matched_ratios=ratios(after), + count=count.tolist(), + sim_fraction=ps.tolist(), + sim_over_data_fraction=(ps / pd).tolist(), + mean_R_cell=rcell.tolist(), + mean_R_snr=(rsum.sum(1) / count.sum(1)).tolist(), + mean_R_size=(rsum.sum(0) / count.sum(0)).tolist(), + summaries=summaries, + ) + maps.append(ps / pd) + cells.append(rcell) + fractions.append(ps) + print( + "GRID", grid, json.dumps(result["grids"][str(grid)]["ratios"]), flush=True + ) + print("MATCHED", grid, json.dumps(ratios(after)), flush=True) + del gm, own + (output / "results.json").write_text(json.dumps(result, indent=2)) + + fig, axes = plt.subplots(2, 4, figsize=(17, 8), constrained_layout=True) + for i, ratio in enumerate(maps): + ax = axes[0, i] + im = ax.pcolormesh( + es, er, np.log10(ratio).T, cmap="RdBu_r", vmin=-1.5, vmax=1.5 + ) + ax.set( + xscale="log", + yscale="log", + xlabel="SNR", + ylabel="T / T_PSF,reconv", + title=f"Grid {i + 1}: selected sim / data fraction", + ) + fig.colorbar(im, ax=axes[0], label="log10 occupancy fraction ratio") + cs, cr = np.sqrt(es[:-1] * es[1:]), np.sqrt(er[:-1] * er[1:]) + for j in (0, 1): + ax = axes[1, j] + coords = cs if j == 0 else cr + axis = 1 if j == 0 else 0 + ax.plot(coords, pd.sum(axis), color="black", lw=2, label="v2 W_COUNT") + for i, ps in enumerate(fractions): + ax.plot(coords, ps.sum(axis), label=f"Grid {i + 1}") + ax.set( + xscale="log", + xlabel="SNR" if j == 0 else "T / T_PSF,reconv", + ylabel="Fraction per log bin", + ) + ax.legend(fontsize=8) + ax = axes[1, j + 2] + for i, ps in enumerate(fractions): + ax.plot(coords, ps.sum(axis) / pd.sum(axis), label=f"Grid {i + 1}") + ax.axhline(1, color="black", ls="--") + ax.set( + xscale="log", + xlabel="SNR" if j == 0 else "T / T_PSF,reconv", + ylabel="Sim / data marginal fraction", + ) + fig.savefig(output / "occupancy.png", dpi=180) + plt.close(fig) + + fig, axes = plt.subplots(2, 4, figsize=(17, 8), constrained_layout=True) + for i, rc in enumerate(cells): + ax = axes[0, i] + im = ax.pcolormesh(es, er, rc.T, cmap="viridis", vmin=0.2, vmax=1.1) + ax.set( + xscale="log", + yscale="log", + xlabel="SNR", + ylabel="T / T_PSF,reconv", + title=f"Grid {i + 1}: unweighted cell mean R_g", + ) + for j, (coords, key) in enumerate(((cs, "mean_R_snr"), (cr, "mean_R_size"))): + axes[1, j].plot( + coords, result["grids"][str(i + 1)][key], label=f"Grid {i + 1}" + ) + fig.colorbar(im, ax=axes[0], label="Mean (R_g11 + R_g22) / 2") + for j in (0, 1): + axes[1, j].set( + xscale="log", + xlabel="SNR" if j == 0 else "T / T_PSF,reconv", + ylabel="Unweighted mean R_g", + ) + axes[1, j].legend() + for j, base in enumerate(("unweighted", "iv_squared")): + ax = axes[1, j + 2] + kind = "shear" if j == 0 else "total" + key = f"{kind}_des_v2_over_{base}" + for name, field, marker in ( + ("Original", "ratios", "o"), + ("Data occupancy", "occupancy_matched_ratios", "s"), + ): + ax.plot( + range(1, 5), + [result["grids"][str(i)][field][key] for i in range(1, 5)], + marker=marker, + label=name, + ) + ax.axhline( + 0.734 / 0.602 if j == 0 else (0.734 - 0.022) / (0.610 - 0.002), + color="black", + ls="--", + label="v2 data", + ) + ax.set( + xlabel="Grid", + ylabel="R_shear DES / unw" if j == 0 else "R_total DES / IV squared", + xticks=range(1, 5), + ) + ax.legend(fontsize=8) + fig.savefig(output / "per_cell_response.png", dpi=180) + plt.close(fig) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(p) + p.add_argument("--output", type=Path, required=True) + args = p.parse_args() + inputs.configure(args) + run(args.output) diff --git a/scripts/validation/response/summarize_response.py b/scripts/validation/response/summarize_response.py new file mode 100644 index 00000000..aba7e7d3 --- /dev/null +++ b/scripts/validation/response/summarize_response.py @@ -0,0 +1,43 @@ +"""Check all 20 case-d products against Martin's reference measurements.""" + +import argparse +import json +from pathlib import Path + + +def summarize(root, output): + output.mkdir(parents=True, exist_ok=True) + results = [ + json.loads((root / f"grid_{i}/validation.json").read_text()) + for i in range(1, 5) + ] + summary = dict( + products=sum(len(r["branches"]) for r in results), + max_abs_R_difference=max( + b["max_abs_R_difference"] for r in results for b in r["branches"].values() + ), + max_relative_W_difference=max( + b["max_relative_W_difference"] + for r in results + for b in r["branches"].values() + ), + max_paired_m_difference=max( + abs(v["new_paired"] - v["martin_paired"]) + for r in results + for v in r["m"].values() + ), + ) + assert summary["products"] == 20 + for key, value in summary.items(): + if key != "products": + assert value <= 1e-14, (key, value) + (output / "summary.json").write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + summarize(args.root, args.output) diff --git a/scripts/validation/response/summarize_sim_vs_data_response.py b/scripts/validation/response/summarize_sim_vs_data_response.py new file mode 100644 index 00000000..1f460653 --- /dev/null +++ b/scripts/validation/response/summarize_sim_vs_data_response.py @@ -0,0 +1,47 @@ +"""Print direct-response diagnostics from sim_vs_data_response.py (allocated node).""" + +import json +import sys +from pathlib import Path + +import numpy as np + +root = Path(sys.argv[1]) +d = json.loads((root / "results.json").read_text()) +es, er = np.array(d["edges_snr"]), np.array(d["edges_size"]) +pd = np.array(d["data_count"], float) +pd /= pd.sum() +print("Data marginals: fractions in first/last five bins, SNR then size") +print( + pd.sum(1)[:5].sum(), pd.sum(1)[-5:].sum(), pd.sum(0)[:5].sum(), pd.sum(0)[-5:].sum() +) +for grid, g in d["grids"].items(): + print("\nGrid", grid, "N", g["selected"]) + print("policy R_shear11/22 R_sel11/22 means(shear,selection)") + for policy, r in g["responses"].items(): + print( + policy, + np.round(r["shear"]["diagonal"], 6), + np.round(r["selection"]["diagonal"], 6), + round(r["shear"]["mean_diagonal"], 6), + round(r["selection"]["mean_diagonal"], 6), + ) + print("summaries", g["summaries"]) + ps = np.array(g["sim_fraction"]) + print( + "Sim marginals first/last five bins", + ps.sum(1)[:5].sum(), + ps.sum(1)[-5:].sum(), + ps.sum(0)[:5].sum(), + ps.sum(0)[-5:].sum(), + ) + print("R_snr", np.round(g["mean_R_snr"], 3)) + print("R_size", np.round(g["mean_R_size"], 3)) + print( + "matched mean responses", + { + k: [round(v[z]["mean_diagonal"], 6) for z in ("shear", "selection")] + for k, v in g["occupancy_matched_responses"].items() + }, + ) +print("Bin5 edges", es[5], er[5], "last5", es[-6], er[-6]) diff --git a/scripts/validation/response/summarize_v2_table.py b/scripts/validation/response/summarize_v2_table.py new file mode 100644 index 00000000..53a84b0f --- /dev/null +++ b/scripts/validation/response/summarize_v2_table.py @@ -0,0 +1,38 @@ +"""Combine production residual m/c across independent simulation grids.""" + +import argparse +import json +from pathlib import Path + +import numpy as np + + +def summarize(root): + rows = {} + for grid in range(1, 5): + product = json.loads((root / f"production_{grid}.json").read_text()) + p = product["weights"]["w_des"] + rows[str(grid)] = dict( + value=[p[k] for k in ("m1", "c1", "m2", "c2")], + error=[p[k + "_err"] for k in ("m1", "c1", "m2", "c2")], + covariance=p["mc_covariance"], + deleted=p["mc_jackknife"], + labels=product["jackknife_tiles"], + clipping=json.loads((root / f"clipping_{grid}.json").read_text()), + ) + values = np.array([r["value"] for r in rows.values()]) + errors = np.array([r["error"] for r in rows.values()]) + weights = errors**-2 + mean = dict( + value=(np.sum(weights * values, axis=0) / weights.sum(axis=0)).tolist(), + error=np.sqrt(1 / weights.sum(axis=0)).tolist(), + ) + result = dict(order=["m1", "c1", "m2", "c2"], grids=rows, combined=mean) + (root / "summary.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(mean, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + summarize(parser.parse_args().root) diff --git a/scripts/validation/response/v2_data_table.py b/scripts/validation/response/v2_data_table.py new file mode 100644 index 00000000..52c2f43d --- /dev/null +++ b/scripts/validation/response/v2_data_table.py @@ -0,0 +1,93 @@ +"""Measure production residual m with the named v2 data table on allocated nodes.""" + +import argparse +import json +from pathlib import Path + +import inputs +import numpy as np +import yaml +from astropy.io import fits +from inputs import EXTERNAL + +from sp_validation import calibration, catalog_builders, image_sim_config, io +from sp_validation.image_sims import ImageSimMBias + + +def run(grid, output): + directory = output / f"grid_{grid}" + clipping = {} + for name in ("1z2z",) + EXTERNAL: + label = f"{name}_grid_{grid}" + source = inputs.BASE / f"grid_{grid}" / label + cfg = yaml.safe_load((source / "config_mask.yaml").read_text()) + cfg = image_sim_config.calibration_config( + cfg, ["w_des"], source="external", product=inputs.PRODUCT + ) + data = io.read_catalogue(source / "shape_catalog_comprehensive_ngmix.fits") + masks, _ = catalog_builders.get_masks_from_config(cfg, data, verbose=False) + pre = catalog_builders.Mask.from_list( + masks, label="combined", verbose=False + )._mask + gm = calibration.metacal_from_config(data, pre, cfg["metacal"]) + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + clipping[name] = {} + for branch in gm.branches: + d = getattr(gm, branch) + sel = gm.mask_dict[branch] + snr, size = ( + d["flux"][sel] / d["flux_err"][sel], + d["T"][sel] / d["Tpsf"][sel], + ) + es, er = gm.W[:2] + outside = (snr < es[0]) | (snr > es[-1]) | (size < er[0]) | (size > er[-1]) + parent_snr = d["flux"] / d["flux_err"] + parent_size = d["T"] / d["Tpsf"] + parent_out = ( + (parent_snr < es[0]) + | (parent_snr > es[-1]) + | (parent_size < er[0]) + | (parent_size > er[-1]) + ) + clipping[name][branch] = dict( + selected=len(snr), + outside=int(outside.sum()), + fraction=float(outside.mean()), + parent=len(parent_snr), + parent_outside=int(parent_out.sum()), + ) + path = directory / label / "shape_catalog_cut_ngmix.fits" + path.parent.mkdir(parents=True, exist_ok=True) + template = inputs.REFERENCE / f"grid_{grid}/des_d" / label / path.name + calibrated, raw, weights, *_ = calibration.get_calibrated_m_c(gm, False) + with fits.open(template) as hdus: + product = fits.HDUList([hdus[0].copy(), hdus[1].copy()]) + for i in range(2): + np.testing.assert_allclose(product[1].data[f"e{i + 1}_uncal"], raw[i]) + product[1].data[f"e{i + 1}"] = calibrated[i] + product[1].data["w_des"] = weights + product.writeto(path, overwrite=True) + calibration.save_response_product(path, gm, tiles) + print(grid, name, cfg, flush=True) + cfg = yaml.safe_load( + (inputs.BASE / f"grid_{grid}/results/m_bias_config.yaml").read_text() + ) + est = ImageSimMBias( + {**cfg, "grids_dir": str(directory), "w_cols": ["w_des"], "pair_match": False} + ) + est.load_catalogs(False) + prod = est.run(False) + print(json.dumps(prod, indent=2), flush=True) + (output / f"production_{grid}.json").write_text(json.dumps(prod, indent=2)) + (output / f"clipping_{grid}.json").write_text(json.dumps(clipping, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(parser) + parser.add_argument("--grid", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + inputs.configure(args) + args.output.mkdir(parents=True, exist_ok=True) + run(args.grid, args.output) diff --git a/scripts/validation/response/validate_response.py b/scripts/validation/response/validate_response.py new file mode 100644 index 00000000..bee96732 --- /dev/null +++ b/scripts/validation/response/validate_response.py @@ -0,0 +1,233 @@ +"""Validate DES case d on Martin's unblinded grids (compute node only). + +python scripts/validation/response/validate_response.py --grid 1 --output /path/to/results +The input shear comes from the grid manifest-derived m_bias_config.yaml. +No blinded catalogue is accepted by this script. +""" + +import argparse +import json +import resource +import subprocess +import time +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +import numpy as np +import yaml +from astropy.io import fits + +from sp_validation import calibration, catalog_builders, io +from sp_validation.catalog import match_catalogs_radec +from sp_validation.image_sims import ImageSimMBias +from sp_validation.response_diagnostics import pair_products, plot_response + + +def run(grid, output): + start = time.monotonic() + base = inputs.BASE / f"grid_{grid}" + reference = inputs.REFERENCE / f"grid_{grid}/des_d" + output.mkdir(parents=True, exist_ok=True) + with open(base / "results/m_bias_config.yaml") as fh: + cfg = yaml.safe_load(fh) + g_in = cfg["shear_amplitude"] + products = {} + summary = { + "grid": grid, + "injected_shear": g_in, + "branches": {}, + "sp_validation_commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip(), + "weight_scheme": "des", + "W_policy": "self-fitted; refit on spatial deletion", + "des_min_count": 0, + "metacal_step": 0.01, + "grid_bins": 20, + "mean_subtraction": False, + "paired_match_radius_deg": cfg["match_radius_deg"], + } + for name in ("1z2z", "1p2z", "1m2z", "1z2p", "1z2m"): + branch_start = time.monotonic() + label = f"{name}_grid_{grid}" + directory = base / label + with open(directory / "config_mask.yaml") as fh: + mask_config = yaml.safe_load(fh) + data = io.read_catalogue(directory / "shape_catalog_comprehensive_ngmix.fits") + masks, _ = catalog_builders.get_masks_from_config( + mask_config, data, verbose=False + ) + pre = catalog_builders.Mask.from_list( + masks, label="combined", verbose=False + )._mask + cm = mask_config["metacal"].copy() + cm.pop("global_R_weight", None) + # Martin's reference uses unpooled cells; production pooling is tested + # separately with the independent data table. + cm.update(weight_scheme="des", w_col="w_des", des_num_bins=20, des_min_count=0) + gm = calibration.metacal_from_config(data, pre, cm) + shear, raw, w, selected, _, _ = calibration.get_calibrated_m_c(gm, False) + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + cut = data[pre][selected] + outdir = output / label + outdir.mkdir(exist_ok=True) + path = outdir / "shape_catalog_cut_ngmix.fits" + refpath = reference / label / path.name + with fits.open(refpath) as hdus: + np.testing.assert_allclose(raw[0], hdus[1].data["e1_uncal"], atol=1e-12) + np.testing.assert_allclose(raw[1], hdus[1].data["e2_uncal"], atol=1e-12) + refR = np.array( + [[hdus[0].header[f"R_{i}{j}"] for j in (1, 2)] for i in (1, 2)] + ) + refRJ = np.array( + [[hdus[0].header[f"RJ_{i}{j}"] for j in (1, 2)] for i in (1, 2)] + ) + w_diff = np.max(abs(w / hdus[1].data["w_des"] - 1)) + hdus[1].data["e1"] = shear[0] + hdus[1].data["e2"] = shear[1] + hdus[1].data["w_iv"] = gm.ns["w_iv"][selected] + hdus[1].data["w_des"] = w + # Keep only the table/header; write the new uncertainty product below. + fits.HDUList([hdus[0].copy(), hdus[1].copy()]).writeto(path, overwrite=True) + with fits.open(path, mode="update") as hdus: + for i in range(2): + for j in range(2): + for prefix, val in [ + ("R", gm.R), + ("R_G", gm.R_shear_global), + ("R_S", gm.R_selection), + ]: + hdus[0].header[ + f"{prefix}{'_' if prefix == 'R' else ''}{i + 1}{j + 1}" + ] = val[i, j] + calibration.save_response_product(path, gm, tiles) + products[name] = dict( + g=shear, + raw=raw, + w=w, + ra=cut["RA"], + dec=cut["Dec"], + ) + summary["branches"][name] = dict( + n=len(w), + R=gm.R.tolist(), + RJ=gm.R_joint.tolist(), + max_abs_R_difference=float(np.max(abs(gm.R - refR))), + max_abs_RJ_difference=float(np.max(abs(gm.R_joint - refRJ))), + max_relative_W_difference=float(w_diff), + branch_counts=gm.branch_counts, + branch_migration=gm.branch_migration, + branch_weight_sums={k: float(v) for k, v in gm.branch_weight_sums.items()}, + seconds=time.monotonic() - branch_start, + ) + if name == "1z2z": + plot_response(gm, tiles, cut["RA"], cut["Dec"], outdir) + pair = pair_products( + cut["RA"], cut["Dec"], w, gm.R_shear, min_sep=1, max_sep=120, nbins=8 + ) + np.savez(outdir / "pair_products.npz", **pair) + fig, axes = plt.subplots(1, 2, figsize=(10, 4)) + for k, comp in enumerate((0, 3)): + ratio = ( + pair["products"][:, comp, comp] + / pair["global_products"][comp, comp] + - 1 + ) + axes[0].plot(pair["theta"], ratio, "o-", label=("11", "22")[k]) + axes[0].axhline(0, color="k", lw=0.5) + axes[0].set( + xscale="log", + xlabel="Separation [arcmin]", + ylabel="Pair Rg product / global product − 1", + ) + axes[0].legend() + for i, j in ((0, 1), (0, 2), (3, 1), (3, 2)): + axes[1].plot( + pair["theta"], + pair["products"][:, i, j] - pair["global_products"][i, j], + "o-", + label=f"{i},{j}", + ) + axes[1].set( + xscale="log", + xlabel="Separation [arcmin]", + ylabel="Off-diagonal pair − global product", + ) + axes[1].legend() + fig.tight_layout() + fig.savefig(output / "pair_response.png", dpi=160) + plt.close(fig) + summary["stage1"] = dict( + note="Shear-only; selection/weight derivatives omitted. No total-response adequacy verdict.", + max_diagonal_fraction=float( + np.nanmax( + abs( + pair["products"][:, [0, 3], [0, 3]] + / pair["global_products"][[0, 3], [0, 3]] + - 1 + ) + ) + ), + ) + with open(reference / "results/m_bias_results.yaml") as fh: + martin = yaml.safe_load(fh)["weights"]["w_des"] + estimator = ImageSimMBias( + { + **cfg, + "grids_dir": str(output), + "num": grid, + "w_cols": ["w_des"], + "pair_match": False, + } + ) + estimator.load_catalogs(False) + production = estimator.run(False) + summary["production_m"] = production + summary["m"] = {} + for comp, (p, m) in enumerate((("1p2z", "1m2z"), ("1z2p", "1z2m"))): + plus, minus = products[p], products[m] + means = [np.average(x["g"][comp], weights=x["w"]) for x in (plus, minus)] + unpaired = (means[0] - means[1]) / (2 * g_in) - 1 + ip, im = match_catalogs_radec( + plus["ra"], + plus["dec"], + minus["ra"], + minus["dec"], + thresh_deg=cfg["match_radius_deg"], + ) + pairw = 0.5 * (plus["w"][ip] + minus["w"][im]) + paired = np.average( + (plus["g"][comp, ip] - minus["g"][comp, im]) / (2 * g_in) - 1, weights=pairw + ) + intersection = ( + np.average(plus["g"][comp, ip], weights=plus["w"][ip]) + - np.average(minus["g"][comp, im], weights=minus["w"][im]) + ) / (2 * g_in) - 1 + np.testing.assert_allclose(production[f"m{comp + 1}"], unpaired, atol=1e-14) + error = production[f"m{comp + 1}_err"] + summary["m"][f"m{comp + 1}"] = dict( + martin_paired=martin[f"m{comp + 1}"], + new_paired=float(paired), + new_unpaired=float(unpaired), + intersection_branch_weighted=float(intersection), + unpaired_refit_W_tile_jk_error=float(error), + ) + summary["seconds"] = time.monotonic() - start + summary["max_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + with open(output / "validation.json", "w") as fh: + json.dump(summary, fh, indent=2) + print(json.dumps(summary, indent=2), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(parser) + parser.add_argument("--grid", type=int, default=1) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + inputs.configure(args) + run(args.grid, args.output) From 0cc2ec2e28b0c05728469c7ae923f2fcd97df806 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:29:25 +0000 Subject: [PATCH 28/36] ruff autofix (format + safe lint fixes) Pushed by the lint gate. --- scripts/validation/wdes_response/diff_jk.py | 6 +- .../validation/wdes_response/mbias_config.py | 2 + scripts/validation/wdes_response/table.py | 85 ++++++--- .../validation/wdes_response/wdes_response.py | 106 +++++++---- workflow/scripts/im_mbias_config.py | 14 +- workflow/scripts/status_sims.py | 170 ++++++++++++------ 6 files changed, 262 insertions(+), 121 deletions(-) diff --git a/scripts/validation/wdes_response/diff_jk.py b/scripts/validation/wdes_response/diff_jk.py index dc4beeba..650db8aa 100644 --- a/scripts/validation/wdes_response/diff_jk.py +++ b/scripts/validation/wdes_response/diff_jk.py @@ -9,6 +9,7 @@ Pairs are (a, b): prints m_b - m_a per weight column, method and component, and writes them to /diff__.yaml. """ + import os import sys @@ -35,7 +36,10 @@ def samples(out_root, scheme): res = mb._m_c_pair(name_p, name_m, comp, verbose=False, method=method) # _jackknife_pair calls m then c for each w_col in order for k, w_col in enumerate(mb.w_cols): - out[(method, w_col, comp + 1)] = (res[w_col]["m"], got[-2 * len(mb.w_cols) + 2 * k]) + out[(method, w_col, comp + 1)] = ( + res[w_col]["m"], + got[-2 * len(mb.w_cols) + 2 * k], + ) return out finally: image_sims._jackknife_std = orig diff --git a/scripts/validation/wdes_response/mbias_config.py b/scripts/validation/wdes_response/mbias_config.py index 41ad52b7..48b61a39 100644 --- a/scripts/validation/wdes_response/mbias_config.py +++ b/scripts/validation/wdes_response/mbias_config.py @@ -1,6 +1,8 @@ """Write an m-bias config for one grid and response weighting, from the workflow's own config with only the paths changed.""" + import sys + import yaml grid, scheme, out_root = sys.argv[1], sys.argv[2], sys.argv[3] diff --git a/scripts/validation/wdes_response/table.py b/scripts/validation/wdes_response/table.py index 5c4e3b75..4c11022a 100644 --- a/scripts/validation/wdes_response/table.py +++ b/scripts/validation/wdes_response/table.py @@ -1,34 +1,65 @@ -import yaml, numpy as np -from scipy.stats import chi2 as C -W="/n23data1/mkilbing/imsim_wdes_response/grid_%d/%s/results/m_bias_results.yaml" -O="/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow/grid_%d/results/m_bias_results.yaml" -def get(y,meth,w): return y["weights"][w] if meth=="split" else y["joint"]["weights"][w] +import numpy as np +import yaml + +W = "/n23data1/mkilbing/imsim_wdes_response/grid_%d/%s/results/m_bias_results.yaml" +O = "/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow/grid_%d/results/m_bias_results.yaml" + + +def get(y, meth, w): + return y["weights"][w] if meth == "split" else y["joint"]["weights"][w] + + # check iv reproduces original -for g in (1,2,3,4): - a=yaml.safe_load(open(W%(g,"iv"))); b=yaml.safe_load(open(O%g)) - d=max(abs(get(a,m,"w_iv")[k]-get(b,m,"w_iv")[k]) for m in ("split","joint") for k in ("m1","m2","m1_err_jk","m2_err_jk")) +for g in (1, 2, 3, 4): + a = yaml.safe_load(open(W % (g, "iv"))) + b = yaml.safe_load(open(O % g)) + d = max( + abs(get(a, m, "w_iv")[k] - get(b, m, "w_iv")[k]) + for m in ("split", "joint") + for k in ("m1", "m2", "m1_err_jk", "m2_err_jk") + ) print(f"iv check grid_{g}: max diff {d:.1e}") -rows=[("w_iv, R w_iv (existing)","iv","w_iv"),("w_iv, R w_iv per branch (fit)","iv_d","w_iv"),("w_iv, R w_iv per branch (boxes)","iv_box","w_iv"),("w_des, R w_iv (existing)","iv","w_des"), - ("w_des, R w_des fixed (c)","des_c","w_des"),("w_des, R w_des per branch (d)","des_d","w_des")] -for meth in ("split","joint"): - print(f"\n== {'R_shear + R_sel' if meth=='split' else 'R_joint'}") - for lab,s,w in rows: - ys=[yaml.safe_load(open(W%(g,s))) for g in (1,2,3,4)] +rows = [ + ("w_iv, R w_iv (existing)", "iv", "w_iv"), + ("w_iv, R w_iv per branch (fit)", "iv_d", "w_iv"), + ("w_iv, R w_iv per branch (boxes)", "iv_box", "w_iv"), + ("w_des, R w_iv (existing)", "iv", "w_des"), + ("w_des, R w_des fixed (c)", "des_c", "w_des"), + ("w_des, R w_des per branch (d)", "des_d", "w_des"), +] +for meth in ("split", "joint"): + print(f"\n== {'R_shear + R_sel' if meth == 'split' else 'R_joint'}") + for lab, s, w in rows: + ys = [yaml.safe_load(open(W % (g, s))) for g in (1, 2, 3, 4)] print(f" {lab}") - for k in ("m1","m2"): - x=np.array([get(y,meth,w)[k] for y in ys]); e=np.array([get(y,meth,w)[k+"_err_jk"] for y in ys]) - iv=1/e**2; mu=(iv*x).sum()/iv.sum(); s_=iv.sum()**-0.5; ch=((x-mu)**2*iv).sum() - per=" ".join(f"{v:+.4f}+/-{u:.4f}" for v,u in zip(x,e)) + for k in ("m1", "m2"): + x = np.array([get(y, meth, w)[k] for y in ys]) + e = np.array([get(y, meth, w)[k + "_err_jk"] for y in ys]) + iv = 1 / e**2 + mu = (iv * x).sum() / iv.sum() + s_ = iv.sum() ** -0.5 + ch = ((x - mu) ** 2 * iv).sum() + per = " ".join(f"{v:+.4f}+/-{u:.4f}" for v, u in zip(x, e)) print(f" {k} {per} mean {mu:+.4f} +/- {s_:.4f} chi2 {ch:.1f}/3") # differences with their own jackknife error (diff_jk.py) -D="/n23data1/mkilbing/imsim_wdes_response/grid_%d/diff_%s_%s.yaml" -for b,a,w in (("iv_d","iv","w_iv"),("iv_box","iv","w_iv"),("des_d","des_c","w_des")): +D = "/n23data1/mkilbing/imsim_wdes_response/grid_%d/diff_%s_%s.yaml" +for b, a, w in ( + ("iv_d", "iv", "w_iv"), + ("iv_box", "iv", "w_iv"), + ("des_d", "des_c", "w_des"), +): print(f"\n== {b} - {a}, weight {w}") - for meth in ("default","joint"): - ys=[yaml.safe_load(open(D%(g,b,a)))[meth][w] for g in (1,2,3,4)] - for k in ("dm1","dm2"): - x=np.array([y[k] for y in ys]); e=np.array([y[k+"_err_jk"] for y in ys]) - iv=1/e**2; mu=(iv*x).sum()/iv.sum(); s_=iv.sum()**-0.5; ch=((x-mu)**2*iv).sum() - per=" ".join(f"{v:+.4f}+/-{u:.4f}" for v,u in zip(x,e)) - print(f" {meth:7s} {k} {per} mean {mu:+.4f} +/- {s_:.4f} chi2 {ch:.1f}/3") + for meth in ("default", "joint"): + ys = [yaml.safe_load(open(D % (g, b, a)))[meth][w] for g in (1, 2, 3, 4)] + for k in ("dm1", "dm2"): + x = np.array([y[k] for y in ys]) + e = np.array([y[k + "_err_jk"] for y in ys]) + iv = 1 / e**2 + mu = (iv * x).sum() / iv.sum() + s_ = iv.sum() ** -0.5 + ch = ((x - mu) ** 2 * iv).sum() + per = " ".join(f"{v:+.4f}+/-{u:.4f}" for v, u in zip(x, e)) + print( + f" {meth:7s} {k} {per} mean {mu:+.4f} +/- {s_:.4f} chi2 {ch:.1f}/3" + ) diff --git a/scripts/validation/wdes_response/wdes_response.py b/scripts/validation/wdes_response/wdes_response.py index 6747acb6..f6497262 100644 --- a/scripts/validation/wdes_response/wdes_response.py +++ b/scripts/validation/wdes_response/wdes_response.py @@ -130,9 +130,7 @@ def mean(data, b, comp): for a, comp in enumerate(("g1", "g2")): for b, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): R_sel[a, b] = (mean(gm.ns, p, comp) - mean(gm.ns, m, comp)) / h2 - R_joint[a, b] = ( - mean(branch[p], p, comp) - mean(branch[m], m, comp) - ) / h2 + R_joint[a, b] = (mean(branch[p], p, comp) - mean(branch[m], m, comp)) / h2 return R_shear + R_sel, R_joint, R_shear, R_sel @@ -170,10 +168,17 @@ def run_branch(sim_dir, out_root): ) cat_gal = {} sp_joint.compute_weights_gatti( - cat_gal, g_uncorr, gm, dat, mask_combined, mask_metacal, + cat_gal, + g_uncorr, + gm, + dat, + mask_combined, + mask_metacal, num_bins=N_BINS, - snr_min=cm["gal_snr_min"], snr_max=cm["gal_snr_max"], - size_ratio_min=cm["gal_rel_size_min"], size_ratio_max=cm["gal_rel_size_max"], + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + size_ratio_min=cm["gal_rel_size_min"], + size_ratio_max=cm["gal_rel_size_max"], ) dat_m = dat[mask_combined._mask] @@ -181,8 +186,13 @@ def run_branch(sim_dir, out_root): # w_des table from the no-shear sample, checked against the catalogue's edges_s, edges_r, table = w_table( - cat_gal["snr"], cat_gal["size_ratio"], g_uncorr[0], g_uncorr[1], - cat_gal["R_g11"], cat_gal["R_g22"], cm, + cat_gal["snr"], + cat_gal["size_ratio"], + g_uncorr[0], + g_uncorr[1], + cat_gal["R_g11"], + cat_gal["R_g22"], + cm, ) snr_ns, size_ns = snr_size(dat_m, "NOSHEAR") w_c, bad_c = eval_table(snr_ns, size_ns, edges_s, edges_r, table) @@ -203,15 +213,20 @@ def run_branch(sim_dir, out_root): # w_iv per branch: C from a fit on the no-shear selection sig2 = 2 * cm["sigma_eps_prior"] ** 2 - C_ns = np.asarray(dat_m["NGMIX_G1_ERR_NOSHEAR"] + dat_m["NGMIX_G2_ERR_NOSHEAR"], float) + C_ns = np.asarray( + dat_m["NGMIX_G1_ERR_NOSHEAR"] + dat_m["NGMIX_G2_ERR_NOSHEAR"], float + ) w_iv_ns = 1 / (sig2 + C_ns) dev = np.max(np.abs(w_iv_ns / gm.ns["w"] - 1)) print(f" w_iv check: max rel dev = {dev:.2e}", flush=True) if dev > 1e-12: raise ValueError(f"{name}: w_iv from G_ERR does not reproduce gm.ns['w']") coef, resid = fit_lnC(snr_ns[ma], size_ns[ma], C_ns[ma]) - print(f" ln C fit (deg {DEG_C}) on S_ns: rms resid {resid.std():.4f}," - f" coef {np.array2string(coef, precision=4)}", flush=True) + print( + f" ln C fit (deg {DEG_C}) on S_ns: rms resid {resid.std():.4f}," + f" coef {np.array2string(coef, precision=4)}", + flush=True, + ) with np.errstate(invalid="ignore"): f_ns = eval_lnC(snr_ns, size_ns, coef) no_ns = ~np.isfinite(f_ns) @@ -225,15 +240,21 @@ def run_branch(sim_dir, out_root): C_b = np.where(no_ns, np.exp(f_b), C_ns * np.exp(f_b - f_ns)) w_iv_d[b] = 1 / (sig2 + C_b) sel = gm.mask_dict[b] - print(f" iv_d {b}: {int(no_ns[sel].sum())} objects on S_{b} without" - " valid no-shear snr/size, fit value used", flush=True) + print( + f" iv_d {b}: {int(no_ns[sel].sum())} objects on S_{b} without" + " valid no-shear snr/size, fit value used", + flush=True, + ) if not np.all(np.isfinite(w_iv_d[b][sel])): raise ValueError(f"{name}: non-finite iv_d weight on S_{b}") ratio = w_iv_d[b][sel] / w_iv_ns[sel] - 1 comp = "g1" if b in ("p1", "m1") else "g2" rho = np.corrcoef(ratio, gm.ns[comp][sel])[0, 1] - print(f" iv_d {b}: w/w_ns - 1 median {np.median(ratio):+.2e}" - f" rms {ratio.std():.2e}, corr with {comp}_ns {rho:+.3f}", flush=True) + print( + f" iv_d {b}: w/w_ns - 1 median {np.median(ratio):+.2e}" + f" rms {ratio.std():.2e}, corr with {comp}_ns {rho:+.3f}", + flush=True, + ) # w_iv per branch: change of C from the mean C per w_des box i_s, i_r = cell_index(snr_ns[ma], size_ns[ma], edges_s, edges_r, like_pandas=True) @@ -260,9 +281,11 @@ def run_branch(sim_dir, out_root): ratio = w_iv_box[b][sel] / w_iv_ns[sel] - 1 comp = "g1" if b in ("p1", "m1") else "g2" rho = np.corrcoef(ratio, gm.ns[comp][sel])[0, 1] - print(f" iv_box {b}: {n_moved} of {len(sel)} change box, {n_empty} in empty" - f" box; w/w_ns - 1 rms {ratio.std():.2e}, corr with {comp}_ns {rho:+.3f}", - flush=True) + print( + f" iv_box {b}: {n_moved} of {len(sel)} change box, {n_empty} in empty" + f" box; w/w_ns - 1 rms {ratio.std():.2e}, corr with {comp}_ns {rho:+.3f}", + flush=True, + ) schemes = { "iv": {b: gm.ns["w"] for b in BRANCHES}, @@ -280,8 +303,11 @@ def run_branch(sim_dir, out_root): RJ_disk = np.array([[hdr[f"RJ_{i}{j}"] for j in (1, 2)] for i in (1, 2)]) e_uncal_disk = np.array([hdul[1].data["e1_uncal"], hdul[1].data["e2_uncal"]]) R_iv, RJ_iv, _, _ = responses(gm, schemes["iv"]) - print(f" iv check: |R - R_disk| = {np.max(np.abs(R_iv - R_disk)):.2e}," - f" |RJ - RJ_disk| = {np.max(np.abs(RJ_iv - RJ_disk)):.2e}", flush=True) + print( + f" iv check: |R - R_disk| = {np.max(np.abs(R_iv - R_disk)):.2e}," + f" |RJ - RJ_disk| = {np.max(np.abs(RJ_iv - RJ_disk)):.2e}", + flush=True, + ) if np.max(np.abs(R_iv - R_disk)) > 1e-8 or np.max(np.abs(RJ_iv - RJ_disk)) > 1e-8: raise ValueError(f"{name}: iv response does not reproduce the catalogue") if np.max(np.abs(e_uncal_disk - g_uncorr)) > 1e-10: @@ -291,14 +317,19 @@ def run_branch(sim_dir, out_root): for scheme, wts in schemes.items(): R, RJ, R_sh, R_se = responses(gm, wts) n_eff = { - b: float(wts[b][gm.mask_dict[b]].sum() ** 2 - / (wts[b][gm.mask_dict[b]] ** 2).sum()) + b: float( + wts[b][gm.mask_dict[b]].sum() ** 2 + / (wts[b][gm.mask_dict[b]] ** 2).sum() + ) for b in BRANCHES } - print(f" {scheme}: R_shear diag {R_sh[0,0]:.4f} {R_sh[1,1]:.4f}" - f" R_sel diag {R_se[0,0]:+.4f} {R_se[1,1]:+.4f}" - f" RJ diag {RJ[0,0]:.4f} {RJ[1,1]:.4f}" - f" N_eff(ns) {n_eff['ns']:.0f} of {len(ma)}", flush=True) + print( + f" {scheme}: R_shear diag {R_sh[0, 0]:.4f} {R_sh[1, 1]:.4f}" + f" R_sel diag {R_se[0, 0]:+.4f} {R_se[1, 1]:+.4f}" + f" RJ diag {RJ[0, 0]:.4f} {RJ[1, 1]:.4f}" + f" N_eff(ns) {n_eff['ns']:.0f} of {len(ma)}", + flush=True, + ) R_jk = np.empty((len(labels_jk), 2, 2)) RJ_jk = np.empty((len(labels_jk), 2, 2)) for k, t in enumerate(labels_jk): @@ -318,17 +349,28 @@ def run_branch(sim_dir, out_root): h0[f"R_G{i}{j}"] = R_sh[i - 1, j - 1] h0[f"R_S{i}{j}"] = R_se[i - 1, j - 1] h0["RWEIGHT"] = (scheme, "weighting of the response (wdes_response.py)") - cols = [fits.Column(name="TILE_ID", array=labels_jk, - format=f"{max(len(t) for t in labels_jk)}A")] + cols = [ + fits.Column( + name="TILE_ID", + array=labels_jk, + format=f"{max(len(t) for t in labels_jk)}A", + ) + ] for key, resp in (("R", R_jk), ("RJ", RJ_jk)): for i in (1, 2): for j in (1, 2): - cols.append(fits.Column(name=f"{key}_{i}{j}", - array=resp[:, i - 1, j - 1], format="D")) + cols.append( + fits.Column( + name=f"{key}_{i}{j}", + array=resp[:, i - 1, j - 1], + format="D", + ) + ) jk = fits.BinTableHDU.from_columns(cols, name="R_JK") hdus = fits.HDUList([hdul[0], hdul[1], jk]) - hdus.writeto(os.path.join(out_dir, "shape_catalog_cut_ngmix.fits"), - overwrite=True) + hdus.writeto( + os.path.join(out_dir, "shape_catalog_cut_ngmix.fits"), overwrite=True + ) def main(): diff --git a/workflow/scripts/im_mbias_config.py b/workflow/scripts/im_mbias_config.py index 9abfba98..9acc9c98 100644 --- a/workflow/scripts/im_mbias_config.py +++ b/workflow/scripts/im_mbias_config.py @@ -43,10 +43,16 @@ def _repo_provenance(repo): if os.path.isfile(snap): with open(snap) as fh: info = yaml.safe_load(fh) or {} - return {"branch": info.get("branch"), "commit": info.get("head"), - "dirty": info.get("dirty"), "snapshot_of": info.get("source")} - return {"branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(repo, "rev-parse", "HEAD")} + return { + "branch": info.get("branch"), + "commit": info.get("head"), + "dirty": info.get("dirty"), + "snapshot_of": info.get("source"), + } + return { + "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _git(repo, "rev-parse", "HEAD"), + } def _sif_revision(): diff --git a/workflow/scripts/status_sims.py b/workflow/scripts/status_sims.py index c9ff7f5c..d4fe08d8 100644 --- a/workflow/scripts/status_sims.py +++ b/workflow/scripts/status_sims.py @@ -32,16 +32,28 @@ # (exp_psf), chunks ngmix into tile_ngmix_1..N and merges the chunks # (tile_merge_cats). Pass --layout to force one; `auto` reads what is on disk. NATIVE_TILE_STAGES = [ - "tile_get_images", "tile_uncompress", "tile_find_exposures", - "tile_merge_headers", "tile_detect", "tile_fake_psf", "tile_vignets", - "tile_ngmix", "tile_make_cat", + "tile_get_images", + "tile_uncompress", + "tile_find_exposures", + "tile_merge_headers", + "tile_detect", + "tile_fake_psf", + "tile_vignets", + "tile_ngmix", + "tile_make_cat", ] NATIVE_EXP_STAGES = ["exp_get_images", "exp_split"] UNIFIED_TILE_STAGES = [ - "tile_get_images", "tile_uncompress", "tile_find_exposures", - "tile_merge_headers", "tile_detect", "tile_vignets", - "tile_ngmix", "tile_merge_cats", "tile_make_cat", + "tile_get_images", + "tile_uncompress", + "tile_find_exposures", + "tile_merge_headers", + "tile_detect", + "tile_vignets", + "tile_ngmix", + "tile_merge_cats", + "tile_make_cat", ] UNIFIED_EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] @@ -91,8 +103,7 @@ def unit_records(unit_dir: Path): # creates the declared log path even for a job its cone never reached. # Counting those as failures turned one dead tile into a column of Xs # across every downstream stage (run_report calls them "not run"). - failed |= {p.stem for p in ldir.glob("*.json") - if p.stat().st_size > 0} + failed |= {p.stem for p in ldir.glob("*.json") if p.stat().st_size > 0} tomb = unit_dir / "cleaned.json" if tomb.exists(): try: @@ -167,8 +178,11 @@ def detect_layout(run_dir: Path) -> str: for unit_dir in units_of(run_dir, kind): done, failed = unit_records(unit_dir) names = done | failed - if any(_CHUNK_RE.match(m) for m in names) or "tile_merge_cats" in names \ - or "exp_psf" in names: + if ( + any(_CHUNK_RE.match(m) for m in names) + or "tile_merge_cats" in names + or "exp_psf" in names + ): return "unified" if "tile_fake_psf" in names: return "native" @@ -225,8 +239,7 @@ def stage_failures(unit_dir: Path, stage: str, n_chunks: int = 1) -> list: return out or ["no reason recorded"] -def report_failures(run_dir: Path, kind: str, stages: list, - n_chunks: int = 1) -> None: +def report_failures(run_dir: Path, kind: str, stages: list, n_chunks: int = 1) -> None: """Every failed unit with its reasons, then the same grouped by reason.""" units = units_of(run_dir, kind) @@ -261,15 +274,15 @@ def report_failures(run_dir: Path, kind: str, stages: list, if not shared: return print(" by reason:") - for (stage, _), (reason, hit) in sorted(shared.items(), - key=lambda kv: -len(kv[1][1])): + for (stage, _), (reason, hit) in sorted( + shared.items(), key=lambda kv: -len(kv[1][1]) + ): print(f" {len(hit):>4}x {STAGE_ABBR[stage]:<8} {short_paths(reason)}") print(f" {', '.join(hit[:8])}" + (" ..." if len(hit) > 8 else "")) print() -def report_finished(run_dir: Path, kind: str, stages: list, - n_chunks: int = 1) -> None: +def report_finished(run_dir: Path, kind: str, stages: list, n_chunks: int = 1) -> None: """--very-short: the abbreviated header, and ONE row below it -- how many units have COMPLETED each step -- instead of one row per unit.""" units = units_of(run_dir, kind) @@ -283,8 +296,10 @@ def report_finished(run_dir: Path, kind: str, stages: list, # of full numbers and looks untouched -- the opposite of the pre- # tombstone behaviour, where it printed zeros and looked emptied. what = "store deleted" if n_cleaned == len(units) else "stores deleted" - print(f" {n_cleaned}/{len(units)} RECLAIMED ({what}); the row below is " - f"what ran, read back from cleaned.json") + print( + f" {n_cleaned}/{len(units)} RECLAIMED ({what}); the row below is " + f"what ran, read back from cleaned.json" + ) header = f"{'':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) print(header) @@ -300,8 +315,9 @@ def report_finished(run_dir: Path, kind: str, stages: list, print(row) -def report_units(run_dir: Path, kind: str, stages: list, *, short: bool = False, - n_chunks: int = 1) -> None: +def report_units( + run_dir: Path, kind: str, stages: list, *, short: bool = False, n_chunks: int = 1 +) -> None: units = units_of(run_dir, kind) if not units: print(f"No {kind} units found under {run_dir / kind}") @@ -332,8 +348,10 @@ def report_units(run_dir: Path, kind: str, stages: list, *, short: bool = False, print(f"{len(units)} {kind} total") for s in stages: c = counts[s] - print(f" {STAGE_ABBR[s]:<8} complete={c['complete']:<4} failed={c['failed']:<4} " - f"not-yet-run={c['missing']}") + print( + f" {STAGE_ABBR[s]:<8} complete={c['complete']:<4} failed={c['failed']:<4} " + f"not-yet-run={c['missing']}" + ) print() @@ -393,23 +411,29 @@ def report_merged(products_dir: Path, cats: list) -> None: """ merged = sorted(products_dir.glob("final_cat_*.hdf5")) if not merged: - print(f"merged hdf5: none in {products_dir} " - f"(run merge_final_cats, or create_final_cat.py -I -c )") + print( + f"merged hdf5: none in {products_dir} " + f"(run merge_final_cats, or create_final_cat.py -I -c )" + ) return for path in merged: try: import h5py except ImportError: - print(f"merged hdf5: {path.name} ({path.stat().st_size / 1e6:.0f} MB)" - " -- install h5py to check its contents") + print( + f"merged hdf5: {path.name} ({path.stat().st_size / 1e6:.0f} MB)" + " -- install h5py to check its contents" + ) return try: with h5py.File(path, "r") as fh: groups = {} + def walk(name, obj): if isinstance(obj, h5py.Dataset): groups[name] = obj.shape[0] + fh.visititems(walk) except OSError as exc: print(f"merged hdf5: {path.name} UNREADABLE ({exc})") @@ -421,6 +445,7 @@ def walk(name, obj): for c in cats: try: from astropy.io import fits + with fits.open(c, memmap=True) as hdus: src_rows += hdus[1].header["NAXIS2"] except Exception: @@ -433,8 +458,10 @@ def walk(name, obj): note = f" MISMATCH: {len(cats)} final_cat files on disk" elif src_rows >= 0 and n_rows != src_rows: note = f" MISMATCH: {src_rows:,} rows in the per-tile catalogues" - print(f"merged hdf5: {path.name} {n_tiles} tiles, {n_rows:,} rows, " - f"{size:.0f} MB{note}") + print( + f"merged hdf5: {path.name} {n_tiles} tiles, {n_rows:,} rows, " + f"{size:.0f} MB{note}" + ) n_file = products_dir / "n_tiles_final.txt" if n_file.exists(): @@ -446,14 +473,19 @@ def walk(name, obj): def _expand(config: dict) -> dict: """Expand $name and ${name} in every value, from the top-level scalar keys, repeatedly (as ShapePipe's run_config.py does for an sp run config).""" - scalars = {k: v for k, v in config.items() - if isinstance(v, (str, int, float)) and not isinstance(v, bool)} + scalars = { + k: v + for k, v in config.items() + if isinstance(v, (str, int, float)) and not isinstance(v, bool) + } def sub(value, depth=0): if isinstance(value, str): - out = re.sub(r"\$\{(\w+)\}|\$(\w+)", - lambda m: str(scalars.get(m.group(1) or m.group(2), - m.group(0))), value) + out = re.sub( + r"\$\{(\w+)\}|\$(\w+)", + lambda m: str(scalars.get(m.group(1) or m.group(2), m.group(0))), + value, + ) return out if out == value or depth >= 5 else sub(out, depth + 1) if isinstance(value, dict): return {k: sub(v, depth) for k, v in value.items()} @@ -473,35 +505,59 @@ def run_dirs_from_config(path: Path): outputs = config.get("outputs") or {} run_dir = outputs.get("run_dir") if not run_dir or "$" in str(run_dir): - raise ValueError(f"{path}: outputs.run_dir is unset or unexpanded " - f"({run_dir!r})") + raise ValueError( + f"{path}: outputs.run_dir is unset or unexpanded ({run_dir!r})" + ) return Path(run_dir), Path(outputs.get("products_dir") or run_dir) def main(argv=None) -> int: p = argparse.ArgumentParser(description=__doc__) - p.add_argument("run_dir", type=Path, nargs="?", - help="run directory; omit when passing -c") - p.add_argument("-c", "--config", type=Path, - help="ShapePipe run config (sp_*.yaml): read outputs.run_dir " - "and outputs.products_dir from it, $name/${name} expanded") - p.add_argument("-s", "--short", action="store_true", - help="per-tile/per-exposure table only, no aggregate " - "'N units total' / per-stage count summary") - p.add_argument("-S", "--very-short", action="store_true", - help="no per-unit table at all -- just 'N/M finished' " - "for tiles and exposures, once each (overrides -s)") - p.add_argument("-p", "--products", type=Path, - help="products root, when it is neither the run dir nor a " - "product/ sibling of it") - p.add_argument("--du", action="store_true", - help="also measure the run directory on disk (a full walk; " - "slow on a live campaign, instant on a reclaimed one)") - p.add_argument("--layout", choices=("auto", "native", "unified"), - default="auto", - help="stage set to report: the sp_validation native " - "prototype, the unified ShapePipe workflow, or auto " - "(default) to read it off the run dir") + p.add_argument( + "run_dir", type=Path, nargs="?", help="run directory; omit when passing -c" + ) + p.add_argument( + "-c", + "--config", + type=Path, + help="ShapePipe run config (sp_*.yaml): read outputs.run_dir " + "and outputs.products_dir from it, $name/${name} expanded", + ) + p.add_argument( + "-s", + "--short", + action="store_true", + help="per-tile/per-exposure table only, no aggregate " + "'N units total' / per-stage count summary", + ) + p.add_argument( + "-S", + "--very-short", + action="store_true", + help="no per-unit table at all -- just 'N/M finished' " + "for tiles and exposures, once each (overrides -s)", + ) + p.add_argument( + "-p", + "--products", + type=Path, + help="products root, when it is neither the run dir nor a " + "product/ sibling of it", + ) + p.add_argument( + "--du", + action="store_true", + help="also measure the run directory on disk (a full walk; " + "slow on a live campaign, instant on a reclaimed one)", + ) + p.add_argument( + "--layout", + choices=("auto", "native", "unified"), + default="auto", + help="stage set to report: the sp_validation native " + "prototype, the unified ShapePipe workflow, or auto " + "(default) to read it off the run dir", + ) args = p.parse_args(argv) if args.config: From 355d1900faa2c0f858f2cf62dd6c040082dba3a6 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 26 Oct 2026 16:02:55 +0100 Subject: [PATCH 29/36] docs: write out the separate normalisation of R_sel and R_joint Each weighted mean is divided by its own weight sum over its own selection, so R is the response of the ratio _w that it calibrates, including the normalisation term of its derivative. Co-Authored-By: Claude Opus 5.5 --- docs/source/response_calibration.md | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/source/response_calibration.md b/docs/source/response_calibration.md index 97fafb0e..b00638cb 100644 --- a/docs/source/response_calibration.md +++ b/docs/source/response_calibration.md @@ -19,6 +19,41 @@ The artificial metacal step h defaults to 0.01; it is distinct from the manifest | c_e / c_g | Raw c_e = A_ns(e); calibrated c_g = inverse(R_total) c_e | | Applied shear | inverse(R_total) e − c_g; omit c_g subtraction for constant-shear simulations | +## Separate normalisation + +Each weighted mean is divided by its own sum of weights, over its own selection and with its own weights. +For the selection and joint responses, with the metacal branches j± of component j: + +```math +R^{\rm sel}_{ij} = \frac{1}{2h} \left[ + \frac{\sum_{S_{j+}} w^{(j+)} e_i^{(\rm ns)}}{\sum_{S_{j+}} w^{(j+)}} + - \frac{\sum_{S_{j-}} w^{(j-)} e_i^{(\rm ns)}}{\sum_{S_{j-}} w^{(j-)}} +\right], +\qquad +R^{\rm joint}_{ij} = \frac{1}{2h} \left[ + \frac{\sum_{S_{j+}} w^{(j+)} e_i^{(j+)}}{\sum_{S_{j+}} w^{(j+)}} + - \frac{\sum_{S_{j-}} w^{(j-)} e_i^{(j-)}}{\sum_{S_{j-}} w^{(j-)}} +\right]. +``` + +Here S_b is the selection made on branch b and w^(b) the weight evaluated on branch b (DES table at that branch's SNR and size, or IV from its G_ERR). +The two denominators differ, since shear moves objects across the cuts and changes their weights. +R_sel differs from R_joint only in the numerator ellipticity: no-shear for R_sel, the branch's own for R_joint. + +The reason is that the estimator applied to data is the ratio ⟨e⟩_w = Σ w e / Σ w. +Its response is the derivative of the ratio, which has a normalisation term: + +```math +\frac{\partial \langle e \rangle_w}{\partial g} += \frac{1}{\sum w} \frac{\partial \sum w e}{\partial g} +- \langle e \rangle_w \, \frac{1}{\sum w} \frac{\partial \sum w}{\partial g}. +``` + +The finite differences of separately normalised means include both terms, so R is the response of the same weighted mean that it calibrates. +A common denominator for both branches drops the second term. +Symmetric pair weights 0.5(w₊ + w₋) on matched objects remove the change of the weight from the numerator, while R keeps it; this is why modern products refuse `pair_match: true`. +The image-simulation m uses the same rule: ⟨e⟩₊ and ⟨e⟩₋ are each normalised by their own population and weights. + ## Configuration and compatibility ```yaml From 847390378a90c96a310dd916709ca0dc2ef62946 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 26 Oct 2026 20:30:20 +0100 Subject: [PATCH 30/36] Fix wdes_response README references --- scripts/validation/wdes_response/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/validation/wdes_response/README.md b/scripts/validation/wdes_response/README.md index 810e2c7e..ed2c61ab 100644 --- a/scripts/validation/wdes_response/README.md +++ b/scripts/validation/wdes_response/README.md @@ -5,10 +5,10 @@ on the no-shear branch, evaluated on each branch's own SNR and size). The production code in `src/sp_validation/response.py` and `calibration.py` reproduces it; these scripts are kept as the reference it was checked against. -They ran on candide in October 2026 against `feat/spv-launcher` (766cc568) in -the image-sim container. `diff_jk.py` and `table.py` use the old m-bias API +`diff_jk.py` and `table.py` use the old m-bias API (`image_sims._jackknife_std`, `results["joint"]`) and do not run on the current -code. Paths are hard-coded to candide. +code. +Paths are hard-coded to candide. - `wdes_response.py`: recalibrate one grid's branches with the weightings iv, des_c, des_d, iv_d, iv_box; writes one cut catalogue per weighting. @@ -17,6 +17,5 @@ code. Paths are hard-coded to candide. - `table.py`: grid table of m1, m2 and the differences. - `job_wdes.sh`: SLURM array over the four grids. -Outputs: `/n23data1/mkilbing/imsim_wdes_response/grid_*//`, read by -`scripts/validation/round2_weights.py`, `v2_data_table.py` and -`validate_response.py`. +Outputs: `/n23data1/mkilbing/imsim_wdes_response/grid_*//`. +The reproduction scripts in `scripts/validation/response/` read these reference products; see that directory's README for commands. From d2787f2591199febff8374eb0dd9145d64e23c12 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 26 Oct 2026 20:42:35 +0100 Subject: [PATCH 31/36] Satisfy the ruff gate: rename an ambiguous name, format a doc snippet Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01114SG5cvfj1gob7zMSQi8A --- docs/source/using_the_catalogues.md | 100 +++++++++++----------- scripts/validation/wdes_response/table.py | 4 +- 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/docs/source/using_the_catalogues.md b/docs/source/using_the_catalogues.md index b6bbfc6b..e051a239 100644 --- a/docs/source/using_the_catalogues.md +++ b/docs/source/using_the_catalogues.md @@ -38,15 +38,15 @@ import numpy as np from astropy.io import fits import matplotlib.pylab as plt -plt.rcParams['font.size'] = 20 +plt.rcParams["font.size"] = 20 # Optional: to compute the example shear–shear correlation functions import treecorr # Catalogue name base and version -cat_base = 'unions_shapepipe' +cat_base = "unions_shapepipe" year = 2022 -version = 'v1.0' +version = "v1.0" ``` ## 1. Base calibrated catalogue @@ -65,23 +65,23 @@ cuts, recalibrate using the extended catalogue (Section 2). ### Open the file and inspect it ```python -cat_name = f'{cat_base}_{year}_{version}.fits' -print(f'Opening base catalogue {cat_name}...') +cat_name = f"{cat_base}_{year}_{version}.fits" +print(f"Opening base catalogue {cat_name}...") hdu_list = fits.open(cat_name) data = hdu_list[1].data -data.dtype.names # column names -len(data) # number of objects -hdu_list[0].header # primary header (global calibration quantities) -hdu_list[1].header # table header (column units) +data.dtype.names # column names +len(data) # number of objects +hdu_list[0].header # primary header (global calibration quantities) +hdu_list[1].header # table header (column units) ``` ### Example 1: magnitude distribution ```python -plt.hist(data['mag'], bins=100, range=(19, 26)) -plt.xlabel('$r$-band magnitude') -_ = plt.ylabel('frequency') +plt.hist(data["mag"], bins=100, range=(19, 26)) +plt.xlabel("$r$-band magnitude") +_ = plt.ylabel("frequency") ``` ### Example 2: shear–shear correlation function @@ -90,18 +90,18 @@ Build a TreeCorr configuration, taking the coordinate units straight from the FITS header so RA and Dec are interpreted correctly: ```python -coord_unit = hdu_list[1].header['TUNIT1'] -if coord_unit != hdu_list[1].header['TUNIT2']: - raise ValueError('Units for RA and Dec should be equal') +coord_unit = hdu_list[1].header["TUNIT1"] +if coord_unit != hdu_list[1].header["TUNIT2"]: + raise ValueError("Units for RA and Dec should be equal") -sep_unit = 'arcmin' +sep_unit = "arcmin" TreeCorrConfig = { - 'ra_units': coord_unit, - 'dec_units': coord_unit, - 'min_sep': '1', - 'max_sep': '200', - 'sep_units': sep_unit, - 'nbins': 20, + "ra_units": coord_unit, + "dec_units": coord_unit, + "min_sep": "1", + "max_sep": "200", + "sep_units": sep_unit, + "nbins": 20, } ``` @@ -110,11 +110,11 @@ minutes): ```python cat_gal = treecorr.Catalog( - ra=data['RA'], - dec=data['Dec'], - g1=data['e1'], - g2=data['e2'], - w=data['w'], + ra=data["RA"], + dec=data["Dec"], + g1=data["e1"], + g2=data["e2"], + w=data["w"], ra_units=coord_unit, dec_units=coord_unit, ) @@ -124,12 +124,12 @@ gg.process(cat_gal, cat_gal) ```python plt.figure(figsize=(14, 8)) -plt.errorbar(gg.meanr, gg.xip, yerr=np.sqrt(gg.varxip), label=r'$\xi_+$') -plt.errorbar(gg.meanr, gg.xim, yerr=np.sqrt(gg.varxim), label=r'$\xi_-$') +plt.errorbar(gg.meanr, gg.xip, yerr=np.sqrt(gg.varxip), label=r"$\xi_+$") +plt.errorbar(gg.meanr, gg.xim, yerr=np.sqrt(gg.varxim), label=r"$\xi_-$") plt.loglog() plt.legend() -plt.xlabel(rf'$\theta$ [{sep_unit}]') -_ = plt.ylabel(r'$\xi_\pm(\theta)$') +plt.xlabel(rf"$\theta$ [{sep_unit}]") +_ = plt.ylabel(r"$\xi_\pm(\theta)$") ``` ## 2. Extended catalogue @@ -154,13 +154,13 @@ Neither function reconstructs missing five-branch inputs after new cuts. ### Open the file ```python -cat_ext_name = f'{cat_base}_extended_{year}_{version}.fits' -print(f'Opening catalogue {cat_ext_name}...') +cat_ext_name = f"{cat_base}_extended_{year}_{version}.fits" +print(f"Opening catalogue {cat_ext_name}...") hdu_list_ext = fits.open(cat_ext_name) data_ext = hdu_list_ext[1].data data_ext.dtype.names # column names -len(data_ext) # number of objects +len(data_ext) # number of objects ``` ### Step 1 — Select the (sub-)sample @@ -178,7 +178,7 @@ mask = np.full(len(data_ext), True) # mask = data_ext['mag'] < 23.5 # r-band magnitude cut n_kept, n_all = np.count_nonzero(mask), len(data_ext) -print(f'Keeping {n_kept}/{n_all} = {n_kept / n_all * 100:.1f}% objects') +print(f"Keeping {n_kept}/{n_all} = {n_kept / n_all * 100:.1f}% objects") data_ext_sub = data_ext[mask] ``` @@ -192,11 +192,11 @@ for the full sample): ```python c = np.empty(2) for comp in (0, 1): - c[comp] = np.average(data_ext_sub[f'e{comp+1}_uncal'], weights=data_ext_sub['w']) + c[comp] = np.average(data_ext_sub[f"e{comp + 1}_uncal"], weights=data_ext_sub["w"]) -print('Additive bias') +print("Additive bias") for comp in (0, 1): - print(f'c_{comp+1} = {c[comp]:.3g}') + print(f"c_{comp + 1} = {c[comp]:.3g}") ``` **Multiplicative bias.** In metacalibration this is a 2×2 matrix @@ -210,9 +210,11 @@ ensemble $R_g$ is computed *after* the sub-sample selection: R_g = np.empty((2, 2)) for idx in (0, 1): for jdx in (0, 1): - R_g[idx, jdx] = np.average(data_ext_sub[f'R_g{idx+1}{jdx+1}'], weights=data_ext_sub['w']) + R_g[idx, jdx] = np.average( + data_ext_sub[f"R_g{idx + 1}{jdx + 1}"], weights=data_ext_sub["w"] + ) -print('Shear response matrix R_g =') +print("Shear response matrix R_g =") print(np.matrix(R_g)) ``` @@ -225,13 +227,13 @@ The following arithmetic is valid only for the original, unchanged population wi R_s = np.empty((2, 2)) for idx in (0, 1): for jdx in (0, 1): - R_s[idx][jdx] = hdu_list_ext[0].header[f'R_S{idx+1}{jdx+1}'] + R_s[idx][jdx] = hdu_list_ext[0].header[f"R_S{idx + 1}{jdx + 1}"] -print('Selection response matrix R_s =') +print("Selection response matrix R_s =") print(np.matrix(R_s)) R = R_g + R_s -print('Total response matrix R =') +print("Total response matrix R =") print(np.matrix(R)) ``` @@ -241,10 +243,12 @@ The observed and true shear are related by $\gamma^\mathrm{obs} = R\,\gamma + c$ Subtract the additive term and multiply by the inverse total response: ```python -e_uncal_minus_c = np.array([ - data_ext['e1_uncal'] - c[0], - data_ext['e2_uncal'] - c[1], -]) +e_uncal_minus_c = np.array( + [ + data_ext["e1_uncal"] - c[0], + data_ext["e2_uncal"] - c[1], + ] +) e_cal = np.linalg.inv(R).dot(e_uncal_minus_c) ``` @@ -253,5 +257,5 @@ shears computed by hand should match those in the base catalogue: ```python print(e_cal[:, 0:3]) -print(data['e1'][0:3], data['e2'][0:3]) +print(data["e1"][0:3], data["e2"][0:3]) ``` diff --git a/scripts/validation/wdes_response/table.py b/scripts/validation/wdes_response/table.py index 4c11022a..4a8be904 100644 --- a/scripts/validation/wdes_response/table.py +++ b/scripts/validation/wdes_response/table.py @@ -2,7 +2,7 @@ import yaml W = "/n23data1/mkilbing/imsim_wdes_response/grid_%d/%s/results/m_bias_results.yaml" -O = "/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow/grid_%d/results/m_bias_results.yaml" +ORIG = "/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow/grid_%d/results/m_bias_results.yaml" def get(y, meth, w): @@ -12,7 +12,7 @@ def get(y, meth, w): # check iv reproduces original for g in (1, 2, 3, 4): a = yaml.safe_load(open(W % (g, "iv"))) - b = yaml.safe_load(open(O % g)) + b = yaml.safe_load(open(ORIG % g)) d = max( abs(get(a, m, "w_iv")[k] - get(b, m, "w_iv")[k]) for m in ("split", "joint") From 2f9bb43e8de33675a11ff75e422768e38559d605 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 27 Oct 2026 21:34:58 +0100 Subject: [PATCH 32/36] Fix weight and response metadata in calibration example Co-Authored-By: Claude Opus 5.5 --- .../examples/demo_calibrate_minimal_cat.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 87ff13d1..58cc5306 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -17,12 +17,15 @@ # %reload_ext autoreload # %autoreload 2 -import numpy as np from astropy.io import fits import sp_validation.catalog as cat from sp_validation import catalog_builders as sp_joint -from sp_validation.calibration import get_calibrated_m_c, metacal_from_config +from sp_validation.calibration import ( + get_calibrated_m_c, + metacal_from_config, + save_response_product, +) # Initialize calibration class instance obj = sp_joint.CalibrateCat() @@ -110,7 +113,7 @@ cat_gal = {} -w_des = sp_joint.compute_weights_gatti( +sp_joint.compute_weights_gatti( cat_gal, g_uncorr, gal_metacal, @@ -143,7 +146,7 @@ # + # Additional quantities -R_shear = np.mean(gal_metacal.R_shear, 2) +R_shear = gal_metacal.R_shear_global ra = cat.get_col(dat, "RA", mask_combined._mask, mask_metacal) dec = cat.get_col(dat, "Dec", mask_combined._mask, mask_metacal) @@ -153,7 +156,6 @@ # + add_cols = [ - "w_iv", "FLUX_RADIUS", "FWHM_IMAGE", "FWHM_WORLD", @@ -168,12 +170,17 @@ add_cols_data = {} for key in add_cols: add_cols_data[key] = dat[key][mask_combined._mask][mask_metacal] +if gal_metacal.weight_scheme != "iv": + add_cols_data["w_iv"] = gal_metacal.ns["w_iv"][mask_metacal] # + add_cols_data["e1_leak_corrected"] = e1_leak_corrected add_cols_data["e2_leak_corrected"] = e2_leak_corrected +for key in ("R_g11", "R_g12", "R_g21", "R_g22"): + add_cols_data[key] = cat_gal[key] + add_cols_data["e1_PSF"] = cat_gal["e1_PSF"] add_cols_data["e2_PSF"] = cat_gal["e2_PSF"] add_cols_data["fwhm_PSF"] = cat.get_col( @@ -201,7 +208,7 @@ output_shape_cat_path, ra, dec, - w_des, + cat_gal["w_des"] if gal_metacal._legacy_response else w, mag=mag, snr=cat_gal["snr"], g=g_corr_mc, @@ -212,10 +219,11 @@ R_select=gal_metacal.R_selection, c=c, c_err=c_err, - w_type="des", + w_type=gal_metacal.weight_scheme or "des", add_cols=add_cols_data, add_header=header, ) +save_response_product(output_shape_cat_path, gal_metacal) # - with open("masks.txt", "w") as f_out: From 6e06e8b1ef8ebdf376fdf8a84d121668f7dc3d42 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 28 Oct 2026 19:50:51 +0100 Subject: [PATCH 33/36] Use configured weights in PSF leakage fits Co-Authored-By: Claude Opus 5.5 --- .../calibrate_comprehensive_cat.py | 1 + .../examples/demo_calibrate_minimal_cat.py | 1 + src/sp_validation/catalog_builders.py | 6 ++-- .../tests/test_leakage_weight_scheme.py | 30 +++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 src/sp_validation/tests/test_leakage_weight_scheme.py diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 76e6fcef..32162456 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -129,6 +129,7 @@ mask_combined, mask_metacal, num_bins=20, + weight_type=gal_metacal.weight_scheme or "des", ) # Compute leakage-corrected ellipticities diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 58cc5306..66ce8d2b 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -133,6 +133,7 @@ mask_combined, mask_metacal, num_bins=20, + weight_type=gal_metacal.weight_scheme or "des", ) # - diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index d81bdcc3..7f34e11e 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -1106,17 +1106,17 @@ def compute_PSF_leakage( mask_combined, mask_metacal, num_bins=20, + weight_type="des", ): - """Compute PSF Leakage.""" + """Compute PSF leakage using the selected calibration weight.""" cat_gal["e1"] = g_corr_mc[0] cat_gal["e2"] = g_corr_mc[1] cat_gal["e1_PSF"] = sp_cat.get_col(dat, "e1_PSF", mask_combined._mask, mask_metacal) cat_gal["e2_PSF"] = sp_cat.get_col(dat, "e2_PSF", mask_combined._mask, mask_metacal) - weight_type = "des" key = f"w_{weight_type}" if key not in cat_gal: - raise KeyError("Key '{key}' not found in cat_gal") + raise KeyError(f"Key '{key}' not found in cat_gal") try: alpha_1, alpha_2 = calibration.get_alpha_leakage_per_object( diff --git a/src/sp_validation/tests/test_leakage_weight_scheme.py b/src/sp_validation/tests/test_leakage_weight_scheme.py new file mode 100644 index 00000000..9732f783 --- /dev/null +++ b/src/sp_validation/tests/test_leakage_weight_scheme.py @@ -0,0 +1,30 @@ +"""The leakage fit must use the science catalogue's weights.""" + +from types import SimpleNamespace + +import numpy as np + +from sp_validation import catalog_builders + + +def test_leakage_uses_iv_weights(monkeypatch): + cat = {"w_des": np.array([1.0, 0.0]), "w_iv": np.array([0.0, 1.0])} + data = {"e1_PSF": np.array([1.0, 2.0]), "e2_PSF": np.array([2.0, 3.0])} + monkeypatch.setattr( + catalog_builders.sp_cat, "get_col", lambda dat, col, *args: dat[col] + ) + + def fit(gal, num_bins, weight_type): + weights = gal[f"w_{weight_type}"] + return np.average(gal["e1"], weights=weights), 0.0 + + monkeypatch.setattr(catalog_builders.calibration, "get_alpha_leakage_per_object", fit) + result = catalog_builders.compute_PSF_leakage( + cat, + (np.array([1.0, 3.0]), np.array([0.0, 0.0])), + data, + SimpleNamespace(_mask=None), + None, + weight_type="iv", + ) + assert result == (3.0, 0.0) From a41b9641ed53417da5291624aefd223cfb36cd30 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 28 Oct 2026 19:57:15 +0100 Subject: [PATCH 34/36] Calibrate pseudo-Cl shear like real-space shear Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/cosmo_val/core.py | 29 ++++++++++----- src/sp_validation/cosmo_val/pseudo_cl.py | 32 ++++++++++++++-- .../tests/test_downstream_calibrated_shear.py | 37 +++++++++++++++++++ src/sp_validation/tests/test_pseudo_cl.py | 13 ++++++- 4 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 src/sp_validation/tests/test_downstream_calibrated_shear.py diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 20838075..104cac3a 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -18,6 +18,7 @@ find_conservative_scale_cut_key, ) from ..statistics import chi2_and_pte +from ..survey import additive_bias from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin from .cosebis import CosebisMixin @@ -664,7 +665,7 @@ def _read_shear_cols(self, ver, *keys): ) return cols[0] if len(cols) == 1 else cols - def _calibrated_g(self, ver): + def _calibrated_g(self, ver, catalog=None): """Calibrated shear components ``(g1, g2)`` for a catalog version. Applies additive-bias subtraction and the multiplicative response: @@ -672,21 +673,31 @@ def _calibrated_g(self, ver): per-component ``R11``/``R22`` (column names in the config); for every other version it is the scalar ``R`` from the config. - Must be called inside a ``self.results[ver].temporarily_read_data()`` - context, since it reads ``dat_shear`` columns. + Without ``catalog``, call inside ``temporarily_read_data()``. With a + catalogue, calculate the same weighted mean directly from its full + population before any tomographic selection. """ - e1, e2 = self._read_shear_cols(ver, "e1_col", "e2_col") - R1, R2 = self._component_responses(ver) - return e1 / R1 - self.c1[ver], e2 / R2 - self.c2[ver] + if catalog is None: + e1, e2 = self._read_shear_cols(ver, "e1_col", "e2_col") + R1, R2 = self._component_responses(ver) + c1, c2 = self.c1[ver], self.c2[ver] + else: + shear = self.cc[ver]["shear"] + e1, e2 = catalog[shear["e1_col"]], catalog[shear["e2_col"]] + R1, R2 = self._component_responses(ver, catalog) + c1, c2 = additive_bias(e1 / R1, e2 / R2, catalog[shear["w_col"]], 1) + return e1 / R1 - c1, e2 / R2 - c2 - def _component_responses(self, ver): + def _component_responses(self, ver, catalog=None): """Resolve responses identically for mean-c production and subtraction. Called within the catalogue's temporarily_read_data context. """ if ver == "DES": - R1 = np.average(self.results[ver].dat_shear[self.cc[ver]["shear"]["R11"]]) - R2 = np.average(self.results[ver].dat_shear[self.cc[ver]["shear"]["R22"]]) + if catalog is None: + catalog = self.results[ver].dat_shear + R1 = np.average(catalog[self.cc[ver]["shear"]["R11"]]) + R2 = np.average(catalog[self.cc[ver]["shear"]["R22"]]) else: R1 = R2 = self.cc[ver]["shear"]["R"] return R1, R2 diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 361369ac..9c5e8eb7 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -90,7 +90,33 @@ def _spectra(part): return {"ELL": ell, "EE": ee, "EB": eb, "BE": eb.copy(), "BB": bb} +class _ShearCatalogue: + """Override shear columns without modifying the source catalogue.""" + + def __init__(self, catalog, columns): + self.catalog = catalog + self.columns = columns + + def __getitem__(self, key): + if isinstance(key, str): + if key in self.columns: + return self.columns[key] + return self.catalog[key] + return _ShearCatalogue( + self.catalog[key], {name: value[key] for name, value in self.columns.items()} + ) + + class PseudoClMixin: + def _pseudo_cl_catalogue(self, ver): + """Read the catalogue with the same calibrated shear used for ξ±.""" + catalog = open_entry(self.cc[ver]["shear"]) + g1, g2 = self._calibrated_g(ver, catalog) + shear = self.cc[ver]["shear"] + return _ShearCatalogue( + catalog, {shear["e1_col"]: g1, shear["e2_col"]: g2} + ) + # ---------------- Pseudo-Cl properties ---------------- # @property def pseudo_cls(self): @@ -227,7 +253,7 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path, tomo_bin_a, tomo_bin_b): ) # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + cat_gal = self._pseudo_cl_catalogue(ver) # Get the tomographic bin cat_gal_a = self._get_tomographic_bin(params, cat_gal, tomo_bin_a) @@ -324,7 +350,7 @@ def calculate_pseudo_cl_catalog(self, ver, out_path, tomo_bin_a, tomo_bin_b): params = get_params_rho_tau(self.cc[ver]) # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + cat_gal = self._pseudo_cl_catalogue(ver) ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog( catalog=cat_gal, params=params, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b @@ -447,7 +473,7 @@ def calculate_pseudo_cl_inka_cov( self.print_cyan(f"Method used: {self.noise_bias_method}") params = get_params_rho_tau(self.cc[ver]) - cat_gal = open_entry(self.cc[ver]["shear"]) + cat_gal = self._pseudo_cl_catalogue(ver) lmin, lmax, b_lmax = spv_pseudo_cl.pseudo_cl_geometry(self.nside) b = self.get_namaster_bin(lmin, lmax, b_lmax) diff --git a/src/sp_validation/tests/test_downstream_calibrated_shear.py b/src/sp_validation/tests/test_downstream_calibrated_shear.py new file mode 100644 index 00000000..2d2a698f --- /dev/null +++ b/src/sp_validation/tests/test_downstream_calibrated_shear.py @@ -0,0 +1,37 @@ +"""Real- and harmonic-space estimators share calibrated shear.""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +from sp_validation.cosmo_val import pseudo_cl +from sp_validation.cosmo_val.core import CosmologyValidation + + +@pytest.mark.parametrize("response,mean_subtracted", [(0.5, False), (1.0, True)]) +def test_pseudo_cl_uses_xi_shear(monkeypatch, response, mean_subtracted): + catalog = np.array( + [(1.0, 2.0, 1.0), (3.0, -1.0, 3.0)], + dtype=[("e1", "f8"), ("e2", "f8"), ("w", "f8")], + ) + if mean_subtracted: + for col in ("e1", "e2"): + catalog[col] -= np.average(catalog[col], weights=catalog["w"]) + obj = CosmologyValidation.__new__(CosmologyValidation) + obj.cc = { + "test": {"shear": {"e1_col": "e1", "e2_col": "e2", "w_col": "w", "R": response}} + } + obj._results = {"test": SimpleNamespace(dat_shear=catalog)} + obj._c1 = {"test": np.average(catalog["e1"] / response, weights=catalog["w"])} + obj._c2 = {"test": np.average(catalog["e2"] / response, weights=catalog["w"])} + monkeypatch.setattr(pseudo_cl, "open_entry", lambda entry: catalog) + xi = obj._calibrated_g("test") + harmonic = obj._pseudo_cl_catalogue("test") + np.testing.assert_allclose(harmonic["e1"], xi[0]) + np.testing.assert_allclose(harmonic["e2"], xi[1]) + if mean_subtracted: + np.testing.assert_allclose(harmonic["e1"], catalog["e1"]) + np.testing.assert_allclose(harmonic["e2"], catalog["e2"]) + else: + np.testing.assert_allclose(xi[0], [-3.0, 1.0]) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 5e4724f0..2e788f9a 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -674,8 +674,12 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): # calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path, monkeypatch): - """Catalog SACC output retains the unwindowed workspace bandpower window.""" + """Catalog SACC round-trip preserves the historical uncentered reference.""" ver = cv._test_version + shear = cv.cc[ver]["shear"] + monkeypatch.setattr( + cv, "_calibrated_g", lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]) + ) cv.cell_method = "catalog" cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} saved = {} @@ -833,7 +837,7 @@ def test_fiducial_pixel_window_applies_only_to_map(): npt.assert_array_equal(fiducial[key], cl) -def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): +def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path, monkeypatch): """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to @@ -841,6 +845,11 @@ def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): (it drops the BE row); we pin the round-tripped table. """ ver = cv._test_version + shear = cv.cc[ver]["shear"] + # This round-trip regression pins the historical uncentered input. + monkeypatch.setattr( + cv, "_calibrated_g", lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]) + ) cv._pseudo_cls = { ver: { "tomo_bin_1_tomo_bin_1": {}, From a883fc10d62e67d573ca3b82ac06f2cf533b2f03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:53:48 +0000 Subject: [PATCH 35/36] ruff autofix (format + safe lint fixes) Pushed by the lint gate. --- src/sp_validation/cosmo_val/pseudo_cl.py | 7 +++---- src/sp_validation/tests/test_leakage_weight_scheme.py | 4 +++- src/sp_validation/tests/test_pseudo_cl.py | 8 ++++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 9c5e8eb7..8dbb1bef 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -103,7 +103,8 @@ def __getitem__(self, key): return self.columns[key] return self.catalog[key] return _ShearCatalogue( - self.catalog[key], {name: value[key] for name, value in self.columns.items()} + self.catalog[key], + {name: value[key] for name, value in self.columns.items()}, ) @@ -113,9 +114,7 @@ def _pseudo_cl_catalogue(self, ver): catalog = open_entry(self.cc[ver]["shear"]) g1, g2 = self._calibrated_g(ver, catalog) shear = self.cc[ver]["shear"] - return _ShearCatalogue( - catalog, {shear["e1_col"]: g1, shear["e2_col"]: g2} - ) + return _ShearCatalogue(catalog, {shear["e1_col"]: g1, shear["e2_col"]: g2}) # ---------------- Pseudo-Cl properties ---------------- # @property diff --git a/src/sp_validation/tests/test_leakage_weight_scheme.py b/src/sp_validation/tests/test_leakage_weight_scheme.py index 9732f783..bc5870d5 100644 --- a/src/sp_validation/tests/test_leakage_weight_scheme.py +++ b/src/sp_validation/tests/test_leakage_weight_scheme.py @@ -18,7 +18,9 @@ def fit(gal, num_bins, weight_type): weights = gal[f"w_{weight_type}"] return np.average(gal["e1"], weights=weights), 0.0 - monkeypatch.setattr(catalog_builders.calibration, "get_alpha_leakage_per_object", fit) + monkeypatch.setattr( + catalog_builders.calibration, "get_alpha_leakage_per_object", fit + ) result = catalog_builders.compute_PSF_leakage( cat, (np.array([1.0, 3.0]), np.array([0.0, 0.0])), diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 2e788f9a..8c68f928 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -678,7 +678,9 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path, monkeypatch): ver = cv._test_version shear = cv.cc[ver]["shear"] monkeypatch.setattr( - cv, "_calibrated_g", lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]) + cv, + "_calibrated_g", + lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]), ) cv.cell_method = "catalog" cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} @@ -848,7 +850,9 @@ def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path, monkeypatch): shear = cv.cc[ver]["shear"] # This round-trip regression pins the historical uncentered input. monkeypatch.setattr( - cv, "_calibrated_g", lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]) + cv, + "_calibrated_g", + lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]), ) cv._pseudo_cls = { ver: { From 3437b49b89f9d50a6c9cc82785262d443bb0f39c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 6 Nov 2026 16:13:51 -0700 Subject: [PATCH 36/36] Stub the calibrated catalogue read in the iNKA merge regression test Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9Gb8hcmKCbSUirKU76 --- .../regression/test_tomo_inka_merge_crosspol_transpose.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py index c3c910ab..66877c42 100644 --- a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py +++ b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py @@ -136,6 +136,10 @@ def test_fresh_cv_merges_tomographic_covariance(tmp_path, monkeypatch): k: np.zeros(2) for k in ("ra_col", "dec_col", "e1_col", "e2_col", "w_col") }, ) + # Shear calibration is not under test: read the stub catalogue as is. + monkeypatch.setattr( + cv, "_pseudo_cl_catalogue", lambda ver: pseudo_cl.open_entry(None) + ) monkeypatch.setattr( pseudo_cl.spv_pseudo_cl, "get_field_and_workspace_from_catalog",