diff --git a/config/calibration/mask_v1.X.10_im_sim.yaml b/config/calibration/mask_v1.X.10_im_sim.yaml new file mode 100644 index 000000000..f159fa4cd --- /dev/null +++ b/config/calibration/mask_v1.X.10_im_sim.yaml @@ -0,0 +1,75 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, including deblended objects, type v1.X.10 + +# General parameters (can also given on command line) +params: + input_path: shape_catalog_comprehensive_ngmix.fits + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 2 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MCAL_TYPES_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: w + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/config/calibration/mask_v1.X.10_im_sim_R_now.yaml b/config/calibration/mask_v1.X.10_im_sim_R_now.yaml new file mode 100644 index 000000000..3244450f5 --- /dev/null +++ b/config/calibration/mask_v1.X.10_im_sim_R_now.yaml @@ -0,0 +1,75 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, including deblended objects, type v1.X.10 + +# General parameters (can also given on command line) +params: + input_path: shape_catalog_comprehensive_ngmix.fits + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 2 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MCAL_TYPES_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: null + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/config/calibration/mask_v2.0.yaml b/config/calibration/mask_v2.0.yaml index 268d333b1..70aa585ff 100644 --- a/config/calibration/mask_v2.0.yaml +++ b/config/calibration/mask_v2.0.yaml @@ -156,5 +156,7 @@ metacal: # Correct relative size for ellipticity? gal_size_corr_ell: False - # Weight for global response matrix, None for unweighted mean - global_R_weight: w + # DES case d: one fixed no-shear W, evaluated on each branch's SNR/size. + weight_scheme: des + w_col: w_des + des_num_bins: 20 diff --git a/docs/source/response_calibration.md b/docs/source/response_calibration.md new file mode 100644 index 000000000..b00638cbb --- /dev/null +++ b/docs/source/response_calibration.md @@ -0,0 +1,114 @@ +# Response, weights and additive bias + +The applied estimator follows Martin Kilbinger's **Separate** definition on slide S1 (6 October 2026, PDF p14, [shared deck](https://docs.google.com/presentation/d/1BdPlRCf742AcpfJVrAe_c3-OOp1ARiwuU9RMNflb0U8/edit)). +DES weights follow **case d** in [`scripts/validation/wdes_response/wdes_response.py`](../../scripts/validation/wdes_response/wdes_response.py), functions `w_table`, `eval_table`, `responses`. +Neither split nor joint response on aligned metacal rows includes detection response. + +For branch b, define A_b(x) = sum(S_b w_b x) / sum(S_b w_b). +Every branch has its own selection and denominator. +The artificial metacal step h defaults to 0.01; it is distinct from the manifest's externally injected simulation shear. + +| Quantity | Definition | +| --- | --- | +| W per cell | mean(trace R_g/2)^2 / mean((e1²+e2²)/2), unweighted raw no-shear moments | +| DES weights | Fit W once on S_ns; evaluate that same table at each branch's own SNR and T/Tpsf | +| IV weights | 1/(2 sigma_eps² + G1_ERR² + G2_ERR²), using each branch's own measured errors | +| R_shear | A_ns((e_j+ − e_j−)/(2h)), all four elements | +| R_sel | (A_j+(e_ns) − A_j−(e_ns))/(2h); includes cuts, weight changes and cell migration | +| R_total / R_joint | Apply R_shear + R_sel; save joint = (A_j+(e_j+) − A_j−(e_j−))/(2h) as a diagnostic | +| c_e / c_g | Raw c_e = A_ns(e); calibrated c_g = inverse(R_total) c_e | +| Applied shear | inverse(R_total) e − c_g; omit c_g subtraction for constant-shear simulations | + +## Separate normalisation + +Each weighted mean is divided by its own sum of weights, over its own selection and with its own weights. +For the selection and joint responses, with the metacal branches j± of component j: + +```math +R^{\rm sel}_{ij} = \frac{1}{2h} \left[ + \frac{\sum_{S_{j+}} w^{(j+)} e_i^{(\rm ns)}}{\sum_{S_{j+}} w^{(j+)}} + - \frac{\sum_{S_{j-}} w^{(j-)} e_i^{(\rm ns)}}{\sum_{S_{j-}} w^{(j-)}} +\right], +\qquad +R^{\rm joint}_{ij} = \frac{1}{2h} \left[ + \frac{\sum_{S_{j+}} w^{(j+)} e_i^{(j+)}}{\sum_{S_{j+}} w^{(j+)}} + - \frac{\sum_{S_{j-}} w^{(j-)} e_i^{(j-)}}{\sum_{S_{j-}} w^{(j-)}} +\right]. +``` + +Here S_b is the selection made on branch b and w^(b) the weight evaluated on branch b (DES table at that branch's SNR and size, or IV from its G_ERR). +The two denominators differ, since shear moves objects across the cuts and changes their weights. +R_sel differs from R_joint only in the numerator ellipticity: no-shear for R_sel, the branch's own for R_joint. + +The reason is that the estimator applied to data is the ratio ⟨e⟩_w = Σ w e / Σ w. +Its response is the derivative of the ratio, which has a normalisation term: + +```math +\frac{\partial \langle e \rangle_w}{\partial g} += \frac{1}{\sum w} \frac{\partial \sum w e}{\partial g} +- \langle e \rangle_w \, \frac{1}{\sum w} \frac{\partial \sum w}{\partial g}. +``` + +The finite differences of separately normalised means include both terms, so R is the response of the same weighted mean that it calibrates. +A common denominator for both branches drops the second term. +Symmetric pair weights 0.5(w₊ + w₋) on matched objects remove the change of the weight from the numerator, while R keeps it; this is why modern products refuse `pair_match: true`. +The image-simulation m uses the same rule: ⟨e⟩₊ and ⟨e⟩₋ are each normalised by their own population and weights. + +## Configuration and compatibility + +```yaml +metacal: + weight_scheme: des + w_col: w_des + des_num_bins: 20 + des_min_count: 3 + additive_correction: true # false for constant-shear simulations +``` + +`metacal_from_config(data, preselection, config['metacal'])` is the common entry point. +A mismatched `w_col` raises an error. +Cut products declare `WGHTSCH`, `RAPPLIED`, `MCSTEP`, `WPOLICY` and `IVPOLICY`. +Consumers must use the matching weight and `R: 1`; the catalogue already received the full matrix. +The configured catalogue reader and image-simulation reader refuse a second R or a different numerator weight on these products. +External catalogues without these metadata keep their existing interpretation. + +At the Python API, `global_R_weight='w'` is deprecated and maps to `weight_scheme='iv'`, including squared G_ERR and weighted selection means. +Versioned release configurations without `weight_scheme` explicitly use the legacy response/c and unsquared-IV policy. +Extraction carries all ten branch G_ERR columns and writes corrected `w_iv` using the public variance helper; legacy calibration reconstructs the historical IV internally. +Modern IV requires sheared-branch errors: regenerate incomplete inputs from the tile-final catalogues and retain those columns in the merge allow-list. +The optional direct-calibration tail of `extract_info.py` refuses because it lacks a shared weight/selection configuration; use the comprehensive product and calibration driver. + +DES cells are right-closed, including internal boundaries. +Finite coordinates outside the table range use the nearest edge cell; nonfinite coordinates get zero weight. +Below `des_min_count`, training moments are pooled over the smallest clipped concentric square reaching that count (or the entire table if necessary). +Counts remain the original, unpooled counts; unpooled empty/invalid cells get zero weight. +A row selected only on a sheared branch still receives a fixed-table weight. + +## Spatial uncertainties and simulation calibration + +`save_response_product` adds W_DES, W_COUNT, W_EDGES, R_JK and R_COV extensions when spatial groups are supplied. +R_JK deletes each shared TILE_ID from every branch and recomputes the selected population and its denominator. +R_COV uses (K−1)/K times the sum of outer products of centered delete-one estimates. +Its ordering is row-major R_shear, R_sel, R_total, R_joint, c_e, c_g (20 elements). +For a self-fitted DES table, the writer refits W on each retained no-shear sample, evaluates it on all five artificial branches and stores the calibrated deleted means in `R_JK.c_g1,c_g2`. +`response_jackknife(groups, refit_w=False)` supplies the conditional fixed-W comparison. +The returned c_err is a weighted error of the raw mean (legacy mode keeps historical scatter); correlated response/c uncertainty belongs in R_COV. + +Image simulations use a named, independent DES data-table product, declared by `image_sim_config.calibration_config` and the image-simulation workflow configuration. +The product contains W, counts and bin edges, with its name recorded in the calibrated catalogue metadata. +Self-fitted weights correlate with the measured shapes: the deterministic test in `test_des_table_independence.py` gives m = −1/N_cell for self-fitting and zero for independent training. +An external table stays fixed during spatial deletions; its training uncertainty is not included. +The production `ImageSimMBias` estimator uses full selected populations with separate branch weights and shared-tile response-recalibrating jackknife covariance for (m1,c1,m2,c2). +Modern products reject pair matching; symmetric pair weights omit a numerator contribution retained in the response and are diagnostic only. +Residual m remains a separate result, not an extra correction applied silently to catalogue e. + +## Diagnostics and reproduction + +`response_diagnostics.plot_response` plots per-object R_g, spatial delete-one R_sel/total/joint, split−joint differences and tile-mean R_g maps. +R_sel is an ensemble derivative, never a per-object response. +`pair_products` measures all 16 angle-binned, pair-weighted products of R_g elements. +`scalar_pair_response` measures _ww/_w²−1 with r=trace(R_g)/2 and shared spatial deletions. +These shear-only diagnostics omit selection/weight derivatives and cannot certify total two-point response adequacy. + +See `scripts/validation/response/README.md` for unblinded case-d reproduction, estimator comparison, independent-table residual m and occupancy-reweighted m. +No generated measurements or plots are stored in git. diff --git a/docs/source/using_the_catalogues.md b/docs/source/using_the_catalogues.md index a3817a3c3..e051a2393 100644 --- a/docs/source/using_the_catalogues.md +++ b/docs/source/using_the_catalogues.md @@ -38,15 +38,15 @@ import numpy as np from astropy.io import fits import matplotlib.pylab as plt -plt.rcParams['font.size'] = 20 +plt.rcParams["font.size"] = 20 # Optional: to compute the example shear–shear correlation functions import treecorr # Catalogue name base and version -cat_base = 'unions_shapepipe' +cat_base = "unions_shapepipe" year = 2022 -version = 'v1.0' +version = "v1.0" ``` ## 1. Base calibrated catalogue @@ -65,23 +65,23 @@ cuts, recalibrate using the extended catalogue (Section 2). ### Open the file and inspect it ```python -cat_name = f'{cat_base}_{year}_{version}.fits' -print(f'Opening base catalogue {cat_name}...') +cat_name = f"{cat_base}_{year}_{version}.fits" +print(f"Opening base catalogue {cat_name}...") hdu_list = fits.open(cat_name) data = hdu_list[1].data -data.dtype.names # column names -len(data) # number of objects -hdu_list[0].header # primary header (global calibration quantities) -hdu_list[1].header # table header (column units) +data.dtype.names # column names +len(data) # number of objects +hdu_list[0].header # primary header (global calibration quantities) +hdu_list[1].header # table header (column units) ``` ### Example 1: magnitude distribution ```python -plt.hist(data['mag'], bins=100, range=(19, 26)) -plt.xlabel('$r$-band magnitude') -_ = plt.ylabel('frequency') +plt.hist(data["mag"], bins=100, range=(19, 26)) +plt.xlabel("$r$-band magnitude") +_ = plt.ylabel("frequency") ``` ### Example 2: shear–shear correlation function @@ -90,18 +90,18 @@ Build a TreeCorr configuration, taking the coordinate units straight from the FITS header so RA and Dec are interpreted correctly: ```python -coord_unit = hdu_list[1].header['TUNIT1'] -if coord_unit != hdu_list[1].header['TUNIT2']: - raise ValueError('Units for RA and Dec should be equal') +coord_unit = hdu_list[1].header["TUNIT1"] +if coord_unit != hdu_list[1].header["TUNIT2"]: + raise ValueError("Units for RA and Dec should be equal") -sep_unit = 'arcmin' +sep_unit = "arcmin" TreeCorrConfig = { - 'ra_units': coord_unit, - 'dec_units': coord_unit, - 'min_sep': '1', - 'max_sep': '200', - 'sep_units': sep_unit, - 'nbins': 20, + "ra_units": coord_unit, + "dec_units": coord_unit, + "min_sep": "1", + "max_sep": "200", + "sep_units": sep_unit, + "nbins": 20, } ``` @@ -110,11 +110,11 @@ minutes): ```python cat_gal = treecorr.Catalog( - ra=data['RA'], - dec=data['Dec'], - g1=data['e1'], - g2=data['e2'], - w=data['w'], + ra=data["RA"], + dec=data["Dec"], + g1=data["e1"], + g2=data["e2"], + w=data["w"], ra_units=coord_unit, dec_units=coord_unit, ) @@ -124,40 +124,43 @@ gg.process(cat_gal, cat_gal) ```python plt.figure(figsize=(14, 8)) -plt.errorbar(gg.meanr, gg.xip, yerr=np.sqrt(gg.varxip), label=r'$\xi_+$') -plt.errorbar(gg.meanr, gg.xim, yerr=np.sqrt(gg.varxim), label=r'$\xi_-$') +plt.errorbar(gg.meanr, gg.xip, yerr=np.sqrt(gg.varxip), label=r"$\xi_+$") +plt.errorbar(gg.meanr, gg.xim, yerr=np.sqrt(gg.varxim), label=r"$\xi_-$") plt.loglog() plt.legend() -plt.xlabel(rf'$\theta$ [{sep_unit}]') -_ = plt.ylabel(r'$\xi_\pm(\theta)$') +plt.xlabel(rf"$\theta$ [{sep_unit}]") +_ = plt.ylabel(r"$\xi_\pm(\theta)$") ``` ## 2. Extended catalogue -This catalogue must be **calibrated before use**. It carries the information -needed for calibration with the method of *metacalibration* -([arXiv:1702.02600](https://arxiv.org/abs/1702.02600)). Cuts may be applied -*before* calibration. +This catalogue must be **calibrated before use**. +A cut catalogue containing only per-object R_g and one global selection response does **not** contain enough information to recalibrate a changed sample or weight scheme. +Use the parent five-branch comprehensive catalogue and `calibration.metacal_from_config` with matching cuts and `weight_scheme: des` (or `iv`). +See [Response definitions](response_calibration.md). +The historical recipe below is retained only to explain the released columns; its reuse of global R_s after new cuts is not a valid new calibration. ```{tip} The `sp_validation` library performs every step below for you: {func}`sp_validation.calibration.get_calibrated_m_c` calibrates an in-memory catalogue for multiplicative and additive bias, and {func}`sp_validation.calibration.get_calibrate_e_from_cat` does the same from a -catalogue on disk. The worked example here is kept for understanding what those -functions compute. +catalogue on disk when its stored weight identity matches. +Unknown/mismatched selection-response metadata is refused, as are changed row counts or weight sums. +Even a historically zero selection response does not establish that it stays zero after new cuts. +Neither function reconstructs missing five-branch inputs after new cuts. ``` ### Open the file ```python -cat_ext_name = f'{cat_base}_extended_{year}_{version}.fits' -print(f'Opening catalogue {cat_ext_name}...') +cat_ext_name = f"{cat_base}_extended_{year}_{version}.fits" +print(f"Opening catalogue {cat_ext_name}...") hdu_list_ext = fits.open(cat_ext_name) data_ext = hdu_list_ext[1].data data_ext.dtype.names # column names -len(data_ext) # number of objects +len(data_ext) # number of objects ``` ### Step 1 — Select the (sub-)sample @@ -175,7 +178,7 @@ mask = np.full(len(data_ext), True) # mask = data_ext['mag'] < 23.5 # r-band magnitude cut n_kept, n_all = np.count_nonzero(mask), len(data_ext) -print(f'Keeping {n_kept}/{n_all} = {n_kept / n_all * 100:.1f}% objects') +print(f"Keeping {n_kept}/{n_all} = {n_kept / n_all * 100:.1f}% objects") data_ext_sub = data_ext[mask] ``` @@ -189,11 +192,11 @@ for the full sample): ```python c = np.empty(2) for comp in (0, 1): - c[comp] = np.average(data_ext_sub[f'e{comp+1}_uncal'], weights=data_ext_sub['w']) + c[comp] = np.average(data_ext_sub[f"e{comp + 1}_uncal"], weights=data_ext_sub["w"]) -print('Additive bias') +print("Additive bias") for comp in (0, 1): - print(f'c_{comp+1} = {c[comp]:.3g}') + print(f"c_{comp + 1} = {c[comp]:.3g}") ``` **Multiplicative bias.** In metacalibration this is a 2×2 matrix @@ -207,28 +210,30 @@ ensemble $R_g$ is computed *after* the sub-sample selection: R_g = np.empty((2, 2)) for idx in (0, 1): for jdx in (0, 1): - R_g[idx, jdx] = np.mean(data_ext_sub[f'R_g{idx+1}{jdx+1}']) + R_g[idx, jdx] = np.average( + data_ext_sub[f"R_g{idx + 1}{jdx + 1}"], weights=data_ext_sub["w"] + ) -print('Shear response matrix R_g =') +print("Shear response matrix R_g =") print(np.matrix(R_g)) ``` The selection response was pre-computed for the *entire* sample and cannot be -obtained per galaxy, so this catalogue version reuses the global -$R_\mathrm{s}$ for sub-samples too. Since $|R_\mathrm{s}| < |R_g|$ the resulting -error is small. +obtained per galaxy. +Do not reuse it for a new selection or weight identity: its small magnitude does not bound the error introduced by doing so. +The following arithmetic is valid only for the original, unchanged population with matching calibration weights. ```python R_s = np.empty((2, 2)) for idx in (0, 1): for jdx in (0, 1): - R_s[idx][jdx] = hdu_list_ext[0].header[f'R_S{idx+1}{jdx+1}'] + R_s[idx][jdx] = hdu_list_ext[0].header[f"R_S{idx + 1}{jdx + 1}"] -print('Selection response matrix R_s =') +print("Selection response matrix R_s =") print(np.matrix(R_s)) R = R_g + R_s -print('Total response matrix R =') +print("Total response matrix R =") print(np.matrix(R)) ``` @@ -238,10 +243,12 @@ The observed and true shear are related by $\gamma^\mathrm{obs} = R\,\gamma + c$ Subtract the additive term and multiply by the inverse total response: ```python -e_uncal_minus_c = np.array([ - data_ext['e1_uncal'] - c[0], - data_ext['e2_uncal'] - c[1], -]) +e_uncal_minus_c = np.array( + [ + data_ext["e1_uncal"] - c[0], + data_ext["e2_uncal"] - c[1], + ] +) e_cal = np.linalg.inv(R).dot(e_uncal_minus_c) ``` @@ -250,5 +257,5 @@ shears computed by hand should match those in the base catalogue: ```python print(e_cal[:, 0:3]) -print(data['e1'][0:3], data['e2'][0:3]) +print(data["e1"][0:3], data["e2"][0:3]) ``` diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index c5248c3e5..321624569 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -25,7 +25,6 @@ import sp_validation.catalog as cat from sp_validation import calibration from sp_validation import catalog_builders as sp_joint -from sp_validation.calibration import metacal # %% # Initialize calibration class instance @@ -81,17 +80,8 @@ cm = config["metacal"] # %% -gal_metacal = metacal( - dat, - mask_combined._mask, - snr_min=cm["gal_snr_min"], - snr_max=cm["gal_snr_max"], - rel_size_min=cm["gal_rel_size_min"], - rel_size_max=cm["gal_rel_size_max"], - size_corr_ell=cm["gal_size_corr_ell"], - sigma_eps=cm["sigma_eps_prior"], - global_R_weight=cm["global_R_weight"], - verbose=True, +gal_metacal = calibration.metacal_from_config( + dat, mask_combined._mask, cm, verbose=True ) # %% @@ -139,6 +129,7 @@ mask_combined, mask_metacal, num_bins=20, + weight_type=gal_metacal.weight_scheme or "des", ) # Compute leakage-corrected ellipticities @@ -176,7 +167,6 @@ # %% # Additional data columns to write to output cat add_cols = [ - "w_iv", "FLUX_RADIUS", "MAGERR_AUTO", "NGMIX_T_NOSHEAR", @@ -188,6 +178,16 @@ add_cols_data[key] = cat.get_col(dat, key, mask_combined._mask, mask_metacal) # %% +# Never copy stale unsquared IV weights from a comprehensive input. +if gal_metacal.weight_scheme != "iv": + add_cols_data["w_iv"] = gal_metacal.ns["w_iv"][mask_metacal] +add_cols_format = {} +if "TILE_ID" in dat.dtype.names: + add_cols_data["TILE_ID"] = np.asarray( + cat.get_col(dat, "TILE_ID", mask_combined._mask, mask_metacal) + ).astype(str) + add_cols_format["TILE_ID"] = f"{max(map(len, add_cols_data['TILE_ID']))}A" + # Additional post-processing columns to write to output cat add_cols_post = [ "R_g11", @@ -227,7 +227,7 @@ output_shape_cat_path, ra, dec, - cat_gal["w_des"], + cat_gal["w_des"] if gal_metacal._legacy_response else w, mag=mag, snr=cat_gal["snr"], g=g_corr_mc, @@ -238,12 +238,20 @@ R_select=gal_metacal.R_selection, c=c, c_err=c_err, - w_type="des", + w_type=gal_metacal.weight_scheme or "des", add_cols=add_cols_data, + add_cols_format=add_cols_format, add_header=header, ) # %% +groups = ( + np.asarray(dat["TILE_ID"][mask_combined._mask]) + if "TILE_ID" in dat.dtype.names + else None +) +calibration.save_response_product(output_shape_cat_path, gal_metacal, groups) + # Write mask information to ASCII file with open("masks.txt", "w") as f_out: for my_mask in masks: diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index dbf256df4..401bba76d 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -36,13 +36,11 @@ # from sp_validation.catalog import * from sp_validation import catalog as spv_cat -from sp_validation import galaxy -from sp_validation.calibration import * +from sp_validation import galaxy, response from sp_validation.calibration import metacal -from sp_validation.galaxy import * -from sp_validation.io import * -from sp_validation.io import read_catalogue -from sp_validation.survey import * +from sp_validation.galaxy import T_to_fwhm, classification_galaxy_overlap_ra_dec +from sp_validation.io import open_stats_file, print_stats, read_catalogue +from sp_validation.survey import get_area, missing_tiles # ## 1. Set-up @@ -238,6 +236,11 @@ for key in add_cols_pre_cal: ext_cols_pre_cal[key] = dd[key] +# Retain upstream branch errors when supplied, even with older params files. +# Historical HDF5 merges omitted them; that must not block DES extraction. +# Modern IV rejects an incomplete set at calibration, rather than freezing W. +ext_cols_pre_cal.update(response.branch_error_columns(dd)) + # Flag to cut duplicate objects in overlapping region with neighbouring tiles ext_cols_pre_cal["overlap"] = cut_overlap add_cols_pre_cal_format["overlap"] = "I" diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index 63dc0828e..6dcc6bd91 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -178,7 +178,16 @@ # Create key names for metacal information prefix = "NGMIX" suffixes = ["1M", "1P", "2M", "2P", "NOSHEAR"] -centers = ["FLAGS", "G1", "G2", "FLUX", "FLUX_ERR", "T", "T_ERR", "T_PSF_RECONV"] +centers = [ + "FLAGS", + "G1", + "G2", + "FLUX", + "FLUX_ERR", + "T", + "T_ERR", + "T_PSF_RECONV", +] for center in centers: for suffix in suffixes: add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") diff --git a/scripts/calibration/stage_image_sim_config.py b/scripts/calibration/stage_image_sim_config.py new file mode 100644 index 000000000..83eda5edb --- /dev/null +++ b/scripts/calibration/stage_image_sim_config.py @@ -0,0 +1,29 @@ +"""Stage a simulation mask config with the independent DES-table policy.""" + +import argparse +from pathlib import Path + +import yaml + +from sp_validation.image_sim_config import calibration_config + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--input", type=Path, required=True) +p.add_argument("--output", type=Path, required=True) +p.add_argument("--w-cols", nargs="+", required=True) +p.add_argument("--source", default="external") +p.add_argument("--product-name", default="") +p.add_argument("--product-path", default="") +a = p.parse_args() +config = calibration_config( + yaml.safe_load(a.input.read_text()), + a.w_cols, + a.source, + dict(name=a.product_name, path=a.product_path) + if a.product_name not in ("", "-") and a.product_path not in ("", "-") + else None, +) +# An earlier workflow may have staged a symlink to a shared mask config. +# Replace that symlink rather than overwriting its target. +a.output.unlink(missing_ok=True) +a.output.write_text(yaml.safe_dump(config, sort_keys=False)) diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 50e52657f..66ce8d2b9 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -17,12 +17,15 @@ # %reload_ext autoreload # %autoreload 2 -import numpy as np from astropy.io import fits import sp_validation.catalog as cat from sp_validation import catalog_builders as sp_joint -from sp_validation.calibration import get_calibrated_m_c, metacal +from sp_validation.calibration import ( + get_calibrated_m_c, + metacal_from_config, + save_response_product, +) # Initialize calibration class instance obj = sp_joint.CalibrateCat() @@ -95,17 +98,7 @@ cm = config["metacal"] -gal_metacal = metacal( - dat, - mask_combined._mask, - snr_min=cm["gal_snr_min"], - snr_max=cm["gal_snr_max"], - rel_size_min=cm["gal_rel_size_min"], - rel_size_max=cm["gal_rel_size_max"], - size_corr_ell=cm["gal_size_corr_ell"], - sigma_eps=cm["sigma_eps_prior"], - verbose=True, -) +gal_metacal = metacal_from_config(dat, mask_combined._mask, cm, verbose=True) # - g_corr_mc, g_uncorr, w, mask_metacal, c, c_err = get_calibrated_m_c(gal_metacal) @@ -120,7 +113,7 @@ cat_gal = {} -w_des = sp_joint.compute_weights_gatti( +sp_joint.compute_weights_gatti( cat_gal, g_uncorr, gal_metacal, @@ -140,6 +133,7 @@ mask_combined, mask_metacal, num_bins=20, + weight_type=gal_metacal.weight_scheme or "des", ) # - @@ -153,7 +147,7 @@ # + # Additional quantities -R_shear = np.mean(gal_metacal.R_shear, 2) +R_shear = gal_metacal.R_shear_global ra = cat.get_col(dat, "RA", mask_combined._mask, mask_metacal) dec = cat.get_col(dat, "Dec", mask_combined._mask, mask_metacal) @@ -163,7 +157,6 @@ # + add_cols = [ - "w_iv", "FLUX_RADIUS", "FWHM_IMAGE", "FWHM_WORLD", @@ -178,12 +171,17 @@ add_cols_data = {} for key in add_cols: add_cols_data[key] = dat[key][mask_combined._mask][mask_metacal] +if gal_metacal.weight_scheme != "iv": + add_cols_data["w_iv"] = gal_metacal.ns["w_iv"][mask_metacal] # + add_cols_data["e1_leak_corrected"] = e1_leak_corrected add_cols_data["e2_leak_corrected"] = e2_leak_corrected +for key in ("R_g11", "R_g12", "R_g21", "R_g22"): + add_cols_data[key] = cat_gal[key] + add_cols_data["e1_PSF"] = cat_gal["e1_PSF"] add_cols_data["e2_PSF"] = cat_gal["e2_PSF"] add_cols_data["fwhm_PSF"] = cat.get_col( @@ -211,7 +209,7 @@ output_shape_cat_path, ra, dec, - w_des, + cat_gal["w_des"] if gal_metacal._legacy_response else w, mag=mag, snr=cat_gal["snr"], g=g_corr_mc, @@ -222,10 +220,11 @@ R_select=gal_metacal.R_selection, c=c, c_err=c_err, - w_type="des", + w_type=gal_metacal.weight_scheme or "des", add_cols=add_cols_data, add_header=header, ) +save_response_product(output_shape_cat_path, gal_metacal) # - with open("masks.txt", "w") as f_out: diff --git a/scripts/shear_calibration+leakage_correction_ellipticities.py b/scripts/shear_calibration+leakage_correction_ellipticities.py index 7d0400529..b3fa2383a 100644 --- a/scripts/shear_calibration+leakage_correction_ellipticities.py +++ b/scripts/shear_calibration+leakage_correction_ellipticities.py @@ -1,110 +1,22 @@ -# Name file: shear_calibration+leakage_correction_ellipticities_.py +"""The matched-cut recalibration recipe is scientifically incomplete. -# @author: Antonin Corinaldi, antonin.corinaldi@cea.fr - -# Python file to perform metacalibration and PSF leakage correction of the ellipticities of matched catalogues -# (cf file get_matched_catalogue.py in src/sp_validation for the creation of the matched catalogues) - - -import numpy as np -from astropy.table import Table -from calibration import fill_cat_gal, get_alpha_leakage_per_object - - -class ResponseMatrix: - def __init__(self, R11, R12, R21, R22): - self.R11 = R11 - self.R12 = R12 - self.R21 = R21 - self.R22 = R22 - - -def get_R(cat, weights=None): - """ - Compute weighted mean response matrix. - """ - R = np.zeros((2, 2)) - - for i in range(2): - for j in range(2): - R[i, j] = np.average(cat[f"R_g{i + 1}{j + 1}"], weights=weights) - - return R +A matched cut catalogue contains only R_g and cannot reconstruct the selection +and weight response of the new population. Calibrate from aligned parent +five-branch inputs via sp_validation.calibration.metacal_from_config instead. +PSF leakage and additive bias are distinct corrections. +""" def process_matched_catalogue( input_path, output_path=None, weight_column="w_iv", num_bins=20, weight_type="iv" ): - """ - Function to: - - read matched catalogue - - compute metacalibration correction - - apply PSF leakage correction - - save corrected matched catalogue - - Parameters - ---------- - input_path : str - Path to input FITS catalogue - output_path : str or None - Path to save output catalogue (if None, overwrite input) - weight_column : str - Column used for weighting (default: w_iv) - num_bins : int - Number of bins for leakage estimation - weight_type : str - Weight type for alpha leakage computation - """ - - # Reading catalogue - table = Table.read(input_path) - - names = [name for name in table.colnames if len(table[name].shape) <= 1] - table = table[names] - - df = table.to_pandas() - - # Computing response matrix - g_uncorr = np.array([df["e1_uncal"], df["e2_uncal"]]) - - gal_metacal = ResponseMatrix(df["R_g11"], df["R_g12"], df["R_g21"], df["R_g22"]) - - cat = {} - fill_cat_gal( - cat, df, g_uncorr, gal_metacal, mask1=None, mask2=None, purpose="weights" - ) - - weights = df[weight_column] - R = get_R(cat, weights=weights) - - # Metacalibration correction - R_inv = np.linalg.inv(R) - e_cal = R_inv.dot(np.array(df[["e1_uncal", "e2_uncal"]].T)) - - df["e1"] = e_cal[0, :] - df["e2"] = e_cal[1, :] - - # PSF leakage correction - alpha_1, alpha_2 = get_alpha_leakage_per_object( - df, num_bins=num_bins, weight_type=weight_type + """Refuse to invent the missing selection response of a matched sample.""" + raise ValueError( + "A matched cut catalogue lacks the five-branch selection/weight " + "response for its changed population. Calibrate the parent with " + "metacal_from_config before using a matched sample diagnostically." ) - df["e1_leak_corrected"] = df["e1"] - alpha_1 * df["e1_PSF"] - df["e2_leak_corrected"] = df["e2"] - alpha_2 * df["e2_PSF"] - - # Save catalogue - output_path = output_path or input_path - - output_table = Table.from_pandas(df) - output_table.write(output_path, overwrite=True) - - print(f"Catalogue processed and saved to: {output_path}") - - return output_table - - -# Example of usage -process_matched_catalogue( - "/n17data/corinaldi/matched_catalogues/unions1.6.9/UNIONS1.6.9_lrg_cross.fits" -) +if __name__ == "__main__": + process_matched_catalogue(None) diff --git a/scripts/validation/response/README.md b/scripts/validation/response/README.md new file mode 100644 index 000000000..e787cd6b8 --- /dev/null +++ b/scripts/validation/response/README.md @@ -0,0 +1,50 @@ +# Unblinded response validation + +Run from the repository root inside the container on an allocated compute node. +Prefix each command with `srun --jobid= -n 1 -c 8 --mem=45G spv-container exec` and set `PYTHONPATH=src`. +Choose an output directory outside git, e.g. `OUT="$PWD/response-results"`. +The commands below use the shared simulation and reference directories and the archived independent DES table by default; no personal scratch inputs are required. + +## Case-d reproduction and estimator comparison + +```bash +for g in 1 2 3 4; do + python scripts/validation/response/validate_response.py --grid "$g" --output "$OUT/martin/grid_$g" +done +python scripts/validation/response/summarize_response.py "$OUT/martin" "$OUT/martin" +python scripts/validation/response/plot_response_m_estimators.py "$OUT/martin" "$OUT/martin" +``` + +The summary asserts agreement in R, W and paired m to 1e-14 on all 20 products, using Martin's unpooled cells (`N_min=0`). +The plot compares symmetric paired, matched branch-weighted and full-population estimators. + +## Independent-table production residual m + +```bash +for g in 1 2 3 4; do + python scripts/validation/response/v2_data_table.py --grid "$g" --output "$OUT/production" +done +python scripts/validation/response/summarize_v2_table.py "$OUT/production" +``` + +## Shear-only and selection-response split + +Measure full-matrix shear-only m and the selection-response contribution with identical numerators and shared-tile deletions, combining four grids with common total-m weights to preserve the split. + +```bash +python scripts/validation/response/m_split.py --output "$OUT/m_split" +``` + +## Simulation/data occupancy and response + +```bash +python scripts/validation/response/sim_vs_data_response.py --output "$OUT/occupancy" +python scripts/validation/response/summarize_sim_vs_data_response.py "$OUT/occupancy" +python scripts/validation/response/occupancy_residual_m.py --output "$OUT/occupancy" --production "$OUT/production" +``` + +Measurement scripts accept `--base` (Martin's comprehensive grids), `--reference` (his calibrated case-d products), `--product` and `--product-name` (independent DES data table). +Their `--help` shows site-specific defaults; the table defaults to the named `v2-d533a078` FITS product at `/n17data/cdaley/unions/validation/2026-10-26_response-425/products/des_weight_product-d533a078.fits`. +That directory's `INDEX.md` records the table's provenance and checksum; use `--product /path/to/des_weight_product-d533a078.fits` to override its location. +FITS, JSON, NPZ and PNG outputs all go under the supplied output directories; no archived review-round outputs are required. +The estimator plot optionally accepts `--production "$OUT/production"` after the production summary exists. diff --git a/scripts/validation/response/inputs.py b/scripts/validation/response/inputs.py new file mode 100644 index 000000000..770676f49 --- /dev/null +++ b/scripts/validation/response/inputs.py @@ -0,0 +1,98 @@ +"""Shared unblinded grid inputs, branch-error join and response helpers.""" + +from pathlib import Path + +import numpy as np +import yaml +from astropy.io import fits +from numpy.lib import recfunctions + +from sp_validation import calibration, catalog_builders, io, response + +BASE = Path( + "/automnt/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow" +) +REFERENCE = Path("/n23data1/mkilbing/imsim_wdes_response") +PRODUCT = dict( + name="v2-d533a078", + path=( + "/n17data/cdaley/unions/validation/2026-10-26_response-425/" + "products/des_weight_product-d533a078.fits" + ), +) +EXTERNAL = ("1p2z", "1m2z", "1z2p", "1z2m") + + +def add_arguments(parser): + parser.add_argument("--base", type=Path, default=BASE) + parser.add_argument("--reference", type=Path, default=REFERENCE) + parser.add_argument("--product", type=Path, default=Path(PRODUCT["path"])) + parser.add_argument("--product-name", default=PRODUCT["name"]) + + +def configure(args): + global BASE, REFERENCE, PRODUCT + BASE, REFERENCE = args.base, args.reference + PRODUCT = dict(name=args.product_name, path=str(args.product)) + + +def load(grid, name, scheme, exact=True, min_count=None): + directory = BASE / f"grid_{grid}/{name}_grid_{grid}" + with open(directory / "config_mask.yaml") as f: + cfg = yaml.safe_load(f) + data = io.read_catalogue(directory / "shape_catalog_comprehensive_ngmix.fits") + if scheme == "iv" and exact: + # Tile finals supply branch errors absent from the comprehensive inputs. + missing = [ + f"NGMIX_G{i}_ERR_{s}" for s in ("1P", "1M", "2P", "2M") for i in (1, 2) + ] + errors = {k: np.full(len(data), np.nan) for k in missing} + tile = np.char.strip(np.asarray(data["TILE_ID"]).astype(str)) + for label in np.unique(tile): + idx = np.flatnonzero(tile == label) + disk_label = f"{float(label):.3f}" + paths = list( + (directory / "product/tiles").glob(f"*/{disk_label}/final_cat-*.fits") + ) + if not paths: + raise FileNotFoundError(f"{directory}: tile {label!r}") + with fits.open(paths[0]) as hdus: + raw = hdus[1].data + order = np.argsort(raw["NUMBER"]) + pos = order[np.searchsorted(raw["NUMBER"][order], data["NUMBER"][idx])] + np.testing.assert_array_equal(raw["NUMBER"][pos], data["NUMBER"][idx]) + for k in ("NGMIX_G1_NOSHEAR", "NGMIX_G2_ERR_NOSHEAR"): + np.testing.assert_allclose( + raw[k][pos], data[k][idx], equal_nan=True + ) + for k in missing: + errors[k][idx] = raw[k][pos] + data = recfunctions.append_fields( + np.asarray(data), list(errors), list(errors.values()), usemask=False + ) + masks, _ = catalog_builders.get_masks_from_config(cfg, data, verbose=False) + pre = catalog_builders.Mask.from_list(masks, label="combined", verbose=False)._mask + cm = cfg["metacal"].copy() + cm.pop("global_R_weight", None) + cm.update(weight_scheme=scheme, w_col=f"w_{scheme}", des_num_bins=20) + if min_count is not None: + cm["des_min_count"] = min_count + gm = calibration.metacal_from_config(data, pre, cm) + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + return gm, tiles + + +def covariance(x): + delta = x - x.mean(axis=0) + return (len(x) - 1) / len(x) * delta.T @ delta + + +def table_weights(gm, table): + return { + b: response.evaluate_w( + table, + getattr(gm, b)["flux"] / getattr(gm, b)["flux_err"], + getattr(gm, b)["T"] / getattr(gm, b)["Tpsf"], + ) + for b in response.BRANCHES + } diff --git a/scripts/validation/response/m_split.py b/scripts/validation/response/m_split.py new file mode 100644 index 000000000..2576166d5 --- /dev/null +++ b/scripts/validation/response/m_split.py @@ -0,0 +1,383 @@ +"""Measure the full-matrix shear/selection calibration split on unblinded grids. + +Run inside the sp_validation container on a Slurm compute node. All writes go +under --output. Inputs follow the sibling response validation scripts. +The numerator is held identical between the shear-only and total calibration. +""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + +import matplotlib +import numpy as np +import yaml + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +from inputs import EXTERNAL + +from sp_validation import calibration, catalog_builders, image_sim_config, io, response +from sp_validation.image_sims import ImageSimMBias + +CONFIGS = ("selection_des", "selection_unweighted", "no_selection_unweighted") +METRICS = ( + "R_shear", + "R_sel", + "m_shear_only", + "m", + "delta_m_sel", + "N_slope", + "scalar_m_shear_only", + "scalar_m", + "scalar_delta_m_sel", +) +ORDER = [f"{metric}_{i}" for metric in METRICS for i in (1, 2)] + + +def covariance(draws): + delta = draws - draws.mean(axis=0) + return (len(draws) - 1) / len(draws) * delta.T @ delta + + +def package(value, draws, labels): + cov = covariance(draws) + return dict( + value=np.asarray(value).tolist(), + error=np.sqrt(np.maximum(0, np.diag(cov))).tolist(), + covariance=cov.tolist(), + deleted=draws.tolist(), + labels=labels.tolist(), + order=ORDER, + ) + + +def estimator(config, cats, matrix): + """Use ImageSimMBias.run, supplying the same in-memory modern catalogue fields + its FITS loader supplies; swap only the applied response for the split. + """ + est = ImageSimMBias({**config, "w_cols": ["none"], "pair_match": False}) + est.cats = {} + for name, cat in cats.items(): + r = cat["full"][matrix] + jr = cat["jk"][matrix] + calibrated = np.linalg.solve(r, cat["raw"]) + est.cats[name] = dict( + weight_scheme="des", + W_policy="fixed-external", + e1=calibrated[0], + e2=calibrated[1], + raw=cat["raw"], + w={"none": cat["weights"]}, + tiles=cat["tiles"], + jk_labels=cat["labels"], + jk_R=jr, + ) + return est.run(False) + + +def run_grid(grid, output, production=None): + cfg_m = yaml.safe_load( + (inputs.BASE / f"grid_{grid}/results/m_bias_config.yaml").read_text() + ) + all_cats = {key: {} for key in CONFIGS} + reference_labels = None + parent_configs = {} + # Only four externally sheared branches enter m; no-external-shear catalogue + # is not used by ImageSimMBias.run for the modern full-population estimator. + for name in EXTERNAL: + directory = inputs.BASE / f"grid_{grid}/{name}_grid_{grid}" + original = yaml.safe_load((directory / "config_mask.yaml").read_text()) + parent_configs[name] = original + data = io.read_catalogue(directory / "shape_catalog_comprehensive_ngmix.fits") + masks, _ = catalog_builders.get_masks_from_config(original, data, verbose=False) + pre = catalog_builders.Mask.from_list( + masks, label="combined", verbose=False + )._mask + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + for key in CONFIGS: + cfg = image_sim_config.calibration_config( + original, ["w_des"], source="external", product=inputs.PRODUCT + ) + if key == "no_selection_unweighted": + cm = cfg["metacal"] + cm.update( + gal_snr_min=-np.inf, + gal_snr_max=np.inf, + gal_rel_size_min=-np.inf, + gal_rel_size_max=np.inf, + ) + gm = calibration.metacal_from_config(data, pre, cfg["metacal"]) + if key != "selection_des": + # Retain selection and per-object responses; use unit weights. + gm.branch_weights = {b: np.ones(len(tiles)) for b in response.BRANCHES} + full = gm.response_estimates() + labels, jk, _ = gm.response_jackknife(tiles, refit_w=False) + if reference_labels is None: + reference_labels = labels + np.testing.assert_array_equal(labels, reference_labels) + idx = gm.mask_dict["ns"] + raw = np.array(gm.branches["ns"])[:, idx] + all_cats[key][name] = dict( + full=full, + jk=jk, + labels=labels, + raw=raw, + weights=gm.branch_weights["ns"][idx], + tiles=tiles[idx], + counts={b: len(gm.mask_dict[b]) for b in response.BRANCHES}, + parent_count=len(tiles), + nonfinite_coordinates={ + b: int( + np.sum( + ~np.isfinite( + getattr(gm, b)["flux"] / getattr(gm, b)["flux_err"] + ) + | ~np.isfinite(getattr(gm, b)["T"] / getattr(gm, b)["Tpsf"]) + ) + ) + for b in response.BRANCHES + }, + ) + print(grid, name, key, all_cats[key][name]["counts"], flush=True) + del gm + del data + result = dict(grid=grid, parent_configs=parent_configs, configurations={}) + for key, cats in all_cats.items(): + total = estimator(cfg_m, cats, "total") + shear = estimator(cfg_m, cats, "shear") + value = np.empty((len(METRICS), 2)) + draws = np.empty((len(reference_labels), len(METRICS), 2)) + for i, (p, m) in enumerate((EXTERNAL[:2], EXTERNAL[2:])): + for dst, field in ((0, "shear"), (1, "selection")): + value[dst, i] = ( + cats[p]["full"][field][i, i] + cats[m]["full"][field][i, i] + ) / 2 + draws[:, dst, i] = ( + cats[p]["jk"][field][:, i, i] + cats[m]["jk"][field][:, i, i] + ) / 2 + value[2, i], value[3, i] = shear[f"m{i + 1}"], total[f"m{i + 1}"] + draws[:, 2, i] = np.array(shear["mc_jackknife"])[:, 2 * i] + draws[:, 3, i] = np.array(total["mc_jackknife"])[:, 2 * i] + value[4, i] = value[3, i] - value[2, i] + draws[:, 4, i] = draws[:, 3, i] - draws[:, 2, i] + gin = cfg_m["shear_amplitude"] + value[5, i] = (cats[p]["full"]["c_e"][i] - cats[m]["full"]["c_e"][i]) / ( + 2 * gin + ) + draws[:, 5, i] = ( + cats[p]["jk"]["c_e"][:, i] - cats[m]["jk"]["c_e"][:, i] + ) / (2 * gin) + for dst, field in ((6, "shear"), (7, "total")): + value[dst, i] = ( + cats[p]["full"]["c_e"][i] / cats[p]["full"][field][i, i] + - cats[m]["full"]["c_e"][i] / cats[m]["full"][field][i, i] + ) / (2 * gin) - 1 + draws[:, dst, i] = ( + cats[p]["jk"]["c_e"][:, i] / cats[p]["jk"][field][:, i, i] + - cats[m]["jk"]["c_e"][:, i] / cats[m]["jk"][field][:, i, i] + ) / (2 * gin) - 1 + value[8, i] = value[7, i] - value[6, i] + draws[:, 8, i] = draws[:, 7, i] - draws[:, 6, i] + entry = package( + value.ravel(), draws.reshape(len(reference_labels), -1), reference_labels + ) + entry["production_estimator"] = total + entry["branch_responses"] = { + name: dict( + full={k: v.tolist() for k, v in c["full"].items()}, + deleted={k: v.tolist() for k, v in c["jk"].items()}, + counts=c["counts"], + parent_count=c["parent_count"], + nonfinite_coordinates=c["nonfinite_coordinates"], + ) + for name, c in cats.items() + } + result["configurations"][key] = entry + if production is not None: + reference = json.loads((production / f"production_{grid}.json").read_text()) + total = result["configurations"]["selection_des"]["production_estimator"] + for name in ("m1", "m2", "m1_err", "m2_err", "mc_covariance", "mc_jackknife"): + np.testing.assert_allclose(total[name], reference[name], rtol=0, atol=1e-12) + result["production_reproduction_max_abs"] = max( + abs(total[k] - reference[k]) for k in ("m1", "m2", "m1_err", "m2_err") + ) + (output / f"grid_{grid}.json").write_text(json.dumps(result, indent=2)) + return result + + +def combine(grids, key, coherent=False): + entries = [g["configurations"][key] for g in grids] + values = np.array([e["value"] for e in entries]) + covs = np.array([e["covariance"] for e in entries]) + variances = np.diagonal(covs, axis1=1, axis2=2) + if coherent: + variances = np.tile(variances[:, 6:8], (1, len(METRICS))) + # Exact-zero selection response has zero covariance: use equal grid weights. + w = np.divide(1, variances, out=np.zeros_like(variances), where=variances > 0) + w[:, w.sum(axis=0) == 0] = 1 + w /= w.sum(axis=0) + cov = np.sum(covs * w[:, :, None] * w[:, None, :], axis=0) + return dict( + value=np.sum(w * values, axis=0).tolist(), + error=np.sqrt(np.maximum(0, np.diag(cov))).tolist(), + covariance=cov.tolist(), + grid_weights=w.tolist(), + order=ORDER, + ) + + +def plot(results, output): + labels = [ + "No SNR/size cuts\nUnweighted", + "SNR/size cuts\nUnweighted", + "SNR/size cuts\nDES (v2 table)", + ] + keys = CONFIGS[::-1] + fig, axes = plt.subplots(1, 2, figsize=(11, 4.4), sharey=True) + for i, ax in enumerate(axes): + for offset, metric, color, marker, label in ( + (-0.20, 2, "#0072B2", "o", "Shear-response only"), + (0, 4, "#D55E00", "s", "Selection contribution"), + (0.20, 3, "#222222", "D", "Total"), + ): + vals = [ + results["combined_common_m_weights"][k]["value"][2 * metric + i] * 1000 + for k in keys + ] + errs = [ + results["combined_common_m_weights"][k]["error"][2 * metric + i] * 1000 + for k in keys + ] + ax.errorbar( + np.arange(3) + offset, + vals, + yerr=errs, + fmt=marker, + color=color, + capsize=3, + label=label, + ms=5, + ) + ax.axhline(0, color=".6", lw=0.8) + ax.set_xticks(range(3), labels) + ax.set_title(f"Component {i + 1}") + ax.grid(axis="y", alpha=0.2) + axes[0].set_ylabel(r"Multiplicative bias $\times 10^3$") + axes[0].legend(fontsize=8) + fig.suptitle("Shear-only + selection contribution = total", fontsize=12) + fig.text( + 0.5, + 0.01, + "Four-grid means; common inverse-variance total-m weights preserve the split. Shared-tile jackknife errors.", + ha="center", + fontsize=8, + ) + fig.tight_layout(rect=(0, 0.04, 1, 0.94)) + fig.savefig(output / "m_split.png", dpi=180) + plt.close(fig) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(parser) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--production", + type=Path, + help="Check against existing per-grid production JSON", + ) + parser.add_argument( + "--reuse", action="store_true", help="Combine already measured grid files only" + ) + args = parser.parse_args() + inputs.configure(args) + args.output.mkdir(parents=True, exist_ok=True) + start_head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip() + grids = [ + json.loads((args.output / f"grid_{i}.json").read_text()) + if args.reuse + else run_grid(i, args.output, args.production) + for i in range(1, 5) + ] + results = dict( + metadata=dict( + checkout_head_start=start_head, + checkout_head_end=subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip(), + product=inputs.PRODUCT, + product_sha256=hashlib.sha256( + Path(inputs.PRODUCT["path"]).read_bytes() + ).hexdigest(), + assumption="Grids treated as independent, conditional on the named fixed external table; no external-table uncertainty or cross-grid tile covariance included.", + combined="Each reported quantity has its own inverse-jackknife-variance grid weights. These marginal means need not sum. combined_common_m_weights uses total-m weights for all quantities, preserving m = m_shear_only + delta_m_sel and used in the figure.", + response_summary="For component i, R_shear/R_sel are the average diagonal i,i across its positive and negative external branches. Every branch matrix and deletion is also saved; numerator calibration uses each branch matrix separately.", + decomposition="Exact production full-matrix counterfactual: same raw weighted population, numerator and deletions; replace R_total by R_shear. Scalar diagonal-only branchwise counterparts are also saved. There is no unique physical numerator split; changing selection also changes N, so delta_m_sel isolates only the calibration-denominator effect within a configuration.", + no_selection="Existing parent config dat cuts retained (FLAGS<=2, overlap=True, N_EPOCH>=2, mag 15..30, MCAL_TYPES_FAIL=0, no-shear PSF g1/g2 != -10); each artificial branch also requires NGMIX_FLAGS_branch=0. SNR/size bounds replaced by +/-infinity: NaN/infinite coordinates can still fail comparisons. Per-branch counts and nonfinite coordinate counts saved. Unit weights on all branches.", + ), + grids=grids, + combined={k: combine(grids, k) for k in CONFIGS}, + combined_common_m_weights={k: combine(grids, k, True) for k in CONFIGS}, + ) + # Keep covariance between configurations too: all use identical tile labels. + cross_order = [f"{key}:{name}" for key in CONFIGS for name in ORDER] + cross_covariances = [] + for grid in grids: + entries = grid["configurations"] + joint_draws = np.concatenate( + [np.asarray(entries[key]["deleted"]) for key in CONFIGS], axis=1 + ) + cross_cov = covariance(joint_draws) + grid["cross_configuration_covariance"] = dict( + order=cross_order, covariance=cross_cov.tolist() + ) + cross_covariances.append(cross_cov) + for key in CONFIGS: + entry = entries[key] + values = np.asarray(entry["value"]) + draws = np.asarray(entry["deleted"]) + np.testing.assert_allclose( + values[8:10], values[6:8] - values[4:6], atol=1e-14 + ) + np.testing.assert_allclose( + draws[:, 8:10], draws[:, 6:8] - draws[:, 4:6], atol=1e-14 + ) + # Confirm no hidden finite-coordinate or fit-flag rejection remains. + for branch in entries["no_selection_unweighted"]["branch_responses"].values(): + assert all(n == branch["parent_count"] for n in branch["counts"].values()) + assert not any(branch["nonfinite_coordinates"].values()) + for method in ("combined", "combined_common_m_weights"): + weights = np.concatenate( + [np.asarray(results[method][key]["grid_weights"]) for key in CONFIGS], + axis=1, + ) + cov = np.sum( + np.asarray(cross_covariances) * weights[:, :, None] * weights[:, None, :], + axis=0, + ) + results[f"{method}_cross_configuration_covariance"] = dict( + order=cross_order, covariance=cov.tolist() + ) + results["metadata"]["N_slope"] = ( + "Raw weighted external-shear slope: (A_plus(e_ns)-A_minus(e_ns))/(2*g_in). This is N/g in the question's notation." + ) + results["metadata"]["no_selection_verified"] = ( + "Every retained parent row passes all five artificial branch masks; nonfinite SNR/size counts are zero on all four grids. R_sel and its deletion covariance are exactly zero." + ) + v = np.array(results["combined"]["selection_des"]["value"])[6:8] + e = np.array(results["combined"]["selection_des"]["error"])[6:8] + print("Production combined m:", v, "+/-", e, flush=True) + (args.output / "results.json").write_text(json.dumps(results, indent=2)) + plot(results, args.output) + for key, entry in results["combined"].items(): + print(key, dict(zip(ORDER, zip(entry["value"], entry["error"]))), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/response/occupancy_residual_m.py b/scripts/validation/response/occupancy_residual_m.py new file mode 100644 index 000000000..57bb0091f --- /dev/null +++ b/scripts/validation/response/occupancy_residual_m.py @@ -0,0 +1,420 @@ +"""Full-population, fixed-v2-table m with fixed parent occupancy multipliers. + +Run on allocated nodes only. Use shared loading and response/JK kernels. +Anchor multipliers to no-external-shear detections and transfer by same-tile +sky position, never NUMBER (a detection sequence, not a cross-branch ID). +External-only detections extend the anchor catalogue on first appearance; +no population is restricted to matched objects. The anchor cell and multiplier +are then frozen across external/artificial branches and tile deletions. +SNR terciles instead use branch-specific SNR selection, including its R_sel. +""" + +import argparse +import json +from pathlib import Path +from unittest.mock import patch + +import matplotlib + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +import numpy as np +import yaml +from inputs import EXTERNAL, covariance, load, table_weights +from scipy.spatial import cKDTree + +from sp_validation import calibration, response + + +def read_grid(grid, name): + original = calibration.metacal_from_config + meta = {} + + def capture(data, pre, cfg): + # Retain reference coordinates even for detections excluded by preselection. + for k in ( + "RA", + "Dec", + "TILE_ID", + "NGMIX_FLUX_NOSHEAR", + "NGMIX_FLUX_ERR_NOSHEAR", + "NGMIX_T_NOSHEAR", + "NGMIX_T_PSF_RECONV_NOSHEAR", + ): + meta[k] = np.asarray(data[k]).copy() + meta["pre"] = np.flatnonzero(pre) + return original(data, pre, cfg) + + with patch.object(calibration, "metacal_from_config", capture): + gm, tiles = load(grid, name, "des", min_count=3) + return gm, tiles, meta + + +def xyz(ra, dec): + r, d = np.deg2rad(ra), np.deg2rad(dec) + return np.column_stack((np.cos(d) * np.cos(r), np.cos(d) * np.sin(r), np.sin(d))) + + +def index(snr, size, table): + return response.cell_index(snr, table[0]) * table[2].shape[1] + response.cell_index( + size, table[1] + ) + + +def meta_cells(meta, table): + with np.errstate(divide="ignore", invalid="ignore"): + snr = meta["NGMIX_FLUX_NOSHEAR"] / meta["NGMIX_FLUX_ERR_NOSHEAR"] + size = meta["NGMIX_T_NOSHEAR"] / meta["NGMIX_T_PSF_RECONV_NOSHEAR"] + return index(snr, size, table), np.isfinite(snr) & np.isfinite(size) + + +def measure(gm, tiles, w, snr_range=None): + masks, rg = gm.mask_dict, gm.R_shear + if snr_range is not None: + lo, hi = snr_range + masks = {} + for b, rows in gm.mask_dict.items(): + v = getattr(gm, b) + s = v["flux"][rows] / v["flux_err"][rows] + masks[b] = rows[(s > lo) & (s <= hi)] + pos = np.searchsorted(gm.mask_dict["ns"], masks["ns"]) + np.testing.assert_array_equal(gm.mask_dict["ns"][pos], masks["ns"]) + rg = gm.R_shear[:, :, pos] + point = response.responses(gm.branches, masks, w, rg, gm._step) + labels, jk, _ = response.jackknife(gm.branches, masks, w, rg, gm._step, tiles) + return point["c_g"], jk["c_g"], labels, {b: len(v) for b, v in masks.items()} + + +def mc(values, gin): + out = [] + for i, (p, m) in enumerate((("1p2z", "1m2z"), ("1z2p", "1z2m"))): + out.extend( + ( + (values[p][..., i] - values[m][..., i]) / (2 * gin) - 1, + (values[p][..., i] + values[m][..., i]) / 2, + ) + ) + return np.array(out).T + + +def package(value, deleted, labels): + cov = covariance(deleted) + return dict( + value=np.asarray(value).tolist(), + error=np.sqrt(np.diag(cov)).tolist(), + covariance=cov.tolist(), + deleted=deleted.tolist(), + labels=labels.tolist(), + ) + + +def combine(rows): + v = np.array([r["value"] for r in rows]) + e = np.array([r["error"] for r in rows]) + w = e**-2 + a = w / w.sum(0) + cov = sum( + np.array(r["covariance"]) * a[i, :, None] * a[i, None, :] + for i, r in enumerate(rows) + ) + return dict( + value=np.sum(a * v, 0).tolist(), + error=np.sqrt(np.diag(cov)).tolist(), + covariance=cov.tolist(), + weights=a.tolist(), + ) + + +def run(output, production): + path = output / "results.json" + previous = json.loads(path.read_text()) + table, _ = calibration.load_des_weight_product(inputs.PRODUCT) + es, er, wd, dc = table + pd = dc / dc.sum() + # Interpolate the W_COUNT CDF uniformly in log SNR inside occupied cells. + cdf = np.r_[0, np.cumsum(pd.sum(1))] + edges = np.exp(np.interp([0, 1 / 3, 2 / 3, 1], cdf, np.log(es))) + ranges = list(zip(edges[:-1], edges[1:])) + result = dict( + order=["m1", "c1", "m2", "c2"], + product=inputs.PRODUCT, + method="Fixed ns-external anchor-cell multipliers; same-tile sky transfer at production match radius, external-only detections extend anchors; no matching restriction. Fixed across artificial/external shears and JK deletions.", + tercile_method="Original (not occupancy-reweighted) populations; branch-specific SNR cuts with recalculated shear and selection responses. Data W_COUNT terciles, log-uniform interpolation within SNR bins.", + snr_tercile_edges=edges.tolist(), + grids={}, + ) + for grid in range(1, 5): + gm0, _, meta0 = read_grid(grid, "1z2z") + cell0, valid0 = meta_cells(meta0, table) + count = np.array(previous["grids"][str(grid)]["count"]) + factor = pd / (count / count.sum()) + ns = gm0.mask_dict["ns"] + np.testing.assert_array_equal( + np.bincount(cell0[meta0["pre"]][ns], minlength=400).reshape(wd.shape), count + ) + del gm0 + # Keep every initial raw detection as a positional anchor. Invalid raw + # coordinates in SNR/size get neutral occupancy, not a fabricated cell. + f0 = np.where(valid0, factor.ravel()[cell0], 1.0) + anchor = {} + tile0 = np.char.strip(meta0["TILE_ID"].astype(str)) + for t in np.unique(tile0): + use = tile0 == t + anchor[t] = ( + xyz(meta0["RA"][use], meta0["Dec"][use]), + f0[use], + np.zeros(use.sum(), bool), + ) + cfg = yaml.safe_load( + (inputs.BASE / f"grid_{grid}/results/m_bias_config.yaml").read_text() + ) + radius = 2 * np.sin(np.deg2rad(cfg["match_radius_deg"]) / 2) + means, deletes = {}, {} + counts, transfers = {}, {} + all_labels = None + for name in EXTERNAL: + gm, tiles, meta = read_grid(grid, name) + idx, valid = meta_cells(meta, table) + fallback = np.where(valid, factor.ravel()[idx], 1.0) + parent_factor = np.ones(len(meta["RA"])) + target_tile = np.char.strip(meta["TILE_ID"].astype(str)) + distance = np.zeros(len(parent_factor)) + fresh = np.zeros(len(parent_factor), bool) + external_origin = np.zeros(len(parent_factor), bool) + for t in np.unique(target_tile): + rows = np.flatnonzero(target_tile == t) + pos = xyz(meta["RA"][rows], meta["Dec"][rows]) + ap, af, external_anchor = anchor[t] + dist, nearest = cKDTree(ap).query(pos, workers=1) + matched = dist <= radius + parent_factor[rows] = np.where(matched, af[nearest], fallback[rows]) + distance[rows] = dist + fresh[rows] = ~matched + external_origin[rows] = ~matched | external_anchor[nearest] + if (~matched).any(): + anchor[t] = ( + np.concatenate((ap, pos[~matched])), + np.r_[af, fallback[rows[~matched]]], + np.r_[external_anchor, np.ones((~matched).sum(), bool)], + ) + density = parent_factor[meta["pre"]] + w = table_weights(gm, table) + policies = dict( + original=(w, None), + occupancy_matched=({b: v * density for b, v in w.items()}, None), + ) + # Sensitivity to the tiny population without any ns-external anchor. + neutral_density = np.where(external_origin[meta["pre"]], 1.0, density) + policies["external_only_neutral_check"] = ( + {b: v * neutral_density for b, v in w.items()}, + None, + ) + # Diagnostic only: quantify the consequence of re-evaluating density + # at each external branch's ns SNR instead of freezing object factors. + policies["external_ns_lookup_diagnostic"] = ( + {b: v * fallback[meta["pre"]] for b, v in w.items()}, + None, + ) + for i, limits in enumerate(ranges): + policies[f"tercile_{i + 1}"] = (w, limits) + for policy, (weights, limits) in policies.items(): + point, jk, labels, n = measure(gm, tiles, weights, limits) + if all_labels is None: + all_labels = labels + np.testing.assert_array_equal(labels, all_labels) + means.setdefault(policy, {})[name] = point + deletes.setdefault(policy, {})[name] = jk + counts.setdefault(policy, {})[name] = n + selected_raw = meta["pre"][gm.mask_dict["ns"]] + transfers[name] = dict( + new_anchors=int(fresh.sum()), + parent=len(fresh), + selected_new_anchors=int(fresh[selected_raw].sum()), + selected=len(selected_raw), + selected_external_origin_DES_weight_fraction=float( + w["ns"][gm.mask_dict["ns"]][external_origin[selected_raw]].sum() + / w["ns"][gm.mask_dict["ns"]].sum() + ), + selected_new_anchor_DES_weight_fraction=float( + w["ns"][gm.mask_dict["ns"]][fresh[selected_raw]].sum() + / w["ns"][gm.mask_dict["ns"]].sum() + ), + selected_match_99_percentile_arcsec=float( + np.rad2deg( + 2 * np.arcsin(np.quantile(distance[selected_raw], 0.99) / 2) + ) + * 3600 + ), + ) + print("loaded", grid, name, transfers[name], flush=True) + row = { + k: package( + mc(v, cfg["shear_amplitude"]), + mc(deletes[k], cfg["shear_amplitude"]), + all_labels, + ) + for k, v in means.items() + } + for k in ("occupancy_matched", "external_ns_lookup_diagnostic"): + row[k + "_minus_original"] = package( + np.array(row[k]["value"]) - row["original"]["value"], + np.array(row[k]["deleted"]) - row["original"]["deleted"], + all_labels, + ) + row["external_only_neutral_minus_matched"] = package( + np.array(row["external_only_neutral_check"]["value"]) + - row["occupancy_matched"]["value"], + np.array(row["external_only_neutral_check"]["deleted"]) + - row["occupancy_matched"]["deleted"], + all_labels, + ) + # Within-grid paired tercile contrast, including cross-tercile covariance. + row["high_minus_low_tercile"] = package( + np.array(row["tercile_3"]["value"]) - row["tercile_1"]["value"], + np.array(row["tercile_3"]["deleted"]) - row["tercile_1"]["deleted"], + all_labels, + ) + prod = json.loads((production / f"production_{grid}.json").read_text()) + p = prod["weights"]["w_des"] + np.testing.assert_array_equal(all_labels, prod["jackknife_tiles"]) + np.testing.assert_allclose( + row["original"]["value"], + [p[k] for k in result["order"]], + atol=1e-12, + rtol=0, + ) + np.testing.assert_allclose( + row["original"]["deleted"], p["mc_jackknife"], atol=1e-12, rtol=0 + ) + row["counts"] = counts + row["anchor_transfer"] = transfers + result["grids"][str(grid)] = row + print( + "GRID", + grid, + "matched", + row["occupancy_matched"]["value"], + "delta", + row["occupancy_matched_minus_original"]["value"], + flush=True, + ) + keys = list(means) + [ + "occupancy_matched_minus_original", + "external_ns_lookup_diagnostic_minus_original", + "external_only_neutral_minus_matched", + "high_minus_low_tercile", + ] + result["means"] = { + k: combine([r[k] for r in result["grids"].values()]) for k in keys + } + # Shift of original-weighted grid mean: same grid weights for both policies, + # with the covariance of paired deletions, not independent mean errors. + a = np.array(result["means"]["original"]["weights"]) + ds = [r["occupancy_matched_minus_original"] for r in result["grids"].values()] + value = np.sum(a * np.array([r["value"] for r in ds]), axis=0) + cov = sum( + np.array(r["covariance"]) * a[i, :, None] * a[i, None, :] + for i, r in enumerate(ds) + ) + result["means"]["shift_at_original_grid_weights"] = dict( + value=value.tolist(), + error=np.sqrt(np.diag(cov)).tolist(), + covariance=cov.tolist(), + ) + result["combination_note"] = ( + "Component-wise inverse variance, assuming independent grids. Paired differences combined using their own inverse variances; also save shift at common original grid weights. W and occupancy factors held fixed in JK; no data-table or density-estimation uncertainty included." + ) + previous["residual_m_occupancy"] = result + path.write_text(json.dumps(previous, indent=2)) + (output / "residual_m_occupancy.json").write_text(json.dumps(result, indent=2)) + fig, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True) + for j, component in enumerate((0, 2)): + ax = axes[0, j] + for k, label, offset in ( + ("original", "Original", -0.07), + ("occupancy_matched", "Data occupancy", 0.07), + ): + vals = [r[k]["value"][component] * 1000 for r in result["grids"].values()] + err = [r[k]["error"][component] * 1000 for r in result["grids"].values()] + ax.errorbar( + np.arange(1, 5) + offset, + vals, + yerr=err, + fmt="o", + capsize=3, + label=label, + ) + ax.axhline(0, color="gray", ls="--") + ax.set(xlabel="Grid", ylabel=f"m{j + 1} × 1000", xticks=range(1, 5)) + ax.legend() + ax = axes[1, j] + for grid, row in result["grids"].items(): + vals = [row[f"tercile_{i}"]["value"][component] * 1000 for i in (1, 2, 3)] + err = [row[f"tercile_{i}"]["error"][component] * 1000 for i in (1, 2, 3)] + ax.errorbar( + np.arange(1, 4) + (int(grid) - 2.5) * 0.035, + vals, + yerr=err, + fmt="o-", + capsize=2, + alpha=0.65, + label=f"Grid {grid}", + ) + vals = [ + result["means"][f"tercile_{i}"]["value"][component] * 1000 + for i in (1, 2, 3) + ] + err = [ + result["means"][f"tercile_{i}"]["error"][component] * 1000 + for i in (1, 2, 3) + ] + ax.errorbar( + range(1, 4), + vals, + yerr=err, + fmt="ks-", + capsize=4, + lw=2, + label="IV grid mean", + ) + ax.axhline(0, color="gray", ls="--") + ax.set( + xlabel="SNR tercile (data-occupancy boundaries)", + ylabel=f"m{j + 1} × 1000", + xticks=range(1, 4), + ) + ax.legend(fontsize=8) + fig.suptitle("Fixed v2 DES table; full populations; shared-tile jackknife") + fig.savefig(output / "residual_m_occupancy.png", dpi=180) + plt.close(fig) + for grid, row in result["grids"].items(): + print("\nRESULT GRID", grid) + for k in keys: + print( + k, + "value", + np.round(np.array(row[k]["value"]) * 1000, 4), + "err", + np.round(np.array(row[k]["error"]) * 1000, 4), + ) + print("\nMEANS") + for k, v in result["means"].items(): + print( + k, + "value", + np.round(np.array(v["value"]) * 1000, 4), + "err", + np.round(np.array(v["error"]) * 1000, 4), + ) + print("SNR tercile boundaries", edges) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(p) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--production", type=Path, required=True) + args = p.parse_args() + inputs.configure(args) + run(args.output, args.production) diff --git a/scripts/validation/response/plot_response_m_estimators.py b/scripts/validation/response/plot_response_m_estimators.py new file mode 100644 index 000000000..a1d735287 --- /dev/null +++ b/scripts/validation/response/plot_response_m_estimators.py @@ -0,0 +1,68 @@ +"""Compare paired, matched branch-weighted and full-population estimators.""" + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + + +def plot(root, output, production=None): + results = [ + json.loads((root / f"grid_{grid}/validation.json").read_text()) + for grid in range(1, 5) + ] + fig, axes = plt.subplots(1, 2, figsize=(11, 4.6), sharex=True) + x = np.arange(1, 5) + for component, ax in enumerate(axes, 1): + rows = [r["m"][f"m{component}"] for r in results] + for offset, key, label in ( + (-0.12, "new_paired", "Matched, symmetric pair weights (Martin)"), + (0, "intersection_branch_weighted", "Matched, separate branch weights"), + (0.12, "new_unpaired", "Full populations, separate branch weights"), + ): + error = ( + [r["unpaired_refit_W_tile_jk_error"] * 1000 for r in rows] + if key == "new_unpaired" + else None + ) + ax.errorbar( + x + offset, + [r[key] * 1000 for r in rows], + yerr=error, + fmt="o", + capsize=3, + label=label, + ) + if production is not None: + data = json.loads((production / "summary.json").read_text())["grids"] + idx = 2 * (component - 1) + ax.errorbar( + x + 0.24, + [data[str(i)]["value"][idx] * 1000 for i in x], + yerr=[data[str(i)]["error"][idx] * 1000 for i in x], + fmt="s", + capsize=3, + label="Full populations, independent v2 table", + ) + ax.axhline(0, color="black", lw=0.8) + ax.set(xticks=x, xlabel="Grid", ylabel=rf"$m_{component}$ ($10^{{-3}}$)") + ax.grid(axis="y", alpha=0.2) + axes[1].legend(frameon=False, fontsize=8) + fig.tight_layout() + output.mkdir(parents=True, exist_ok=True) + fig.savefig(output / "m_estimators.png", dpi=180, bbox_inches="tight") + plt.close(fig) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--production", type=Path) + args = parser.parse_args() + plot(args.root, args.output, args.production) diff --git a/scripts/validation/response/sim_vs_data_response.py b/scripts/validation/response/sim_vs_data_response.py new file mode 100644 index 000000000..76e7bb5ff --- /dev/null +++ b/scripts/validation/response/sim_vs_data_response.py @@ -0,0 +1,263 @@ +"""Direct ns response and occupancy comparison; run on an allocated node only. + +Use the shared exact-error join, selection and independent-table evaluation. Density matching uses the ns cell's +p_data/p_sim as a fixed parent-object multiplier on every artificial branch: +it changes the population without adding an artificial density-weight derivative. +DES W and IV errors still vary with each artificial branch as in production. +""" + +import argparse +import json +from pathlib import Path +from unittest.mock import patch + +import matplotlib + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +import numpy as np +from inputs import load, table_weights + +from sp_validation import calibration, response + + +def pack(gm, weights): + r = response.responses(gm.branches, gm.mask_dict, weights, gm.R_shear, gm._step) + return { + k: dict( + matrix=r[k].tolist(), + diagonal=np.diag(r[k]).tolist(), + mean_diagonal=float(np.trace(r[k]) / 2), + ) + for k in ("shear", "selection", "total", "joint") + } + + +def ratios(results): + def r(a, b, kind): + return results[a][kind]["mean_diagonal"] / results[b][kind]["mean_diagonal"] + + return { + f"{kind}_{des}_over_{base}": r(des, base, kind) + for kind in ("shear", "total") + for des in ("des_v2", "des_self") + for base in ("unweighted", "iv_squared") + } + + +def run(output): + output.mkdir(parents=True, exist_ok=True) + table, nmin = calibration.load_des_weight_product(inputs.PRODUCT) + es, er, wd, data_count = table + pd = data_count / data_count.sum() + result = dict( + product=inputs.PRODUCT, + data_count=data_count.tolist(), + edges_snr=es.tolist(), + edges_size=er.tolist(), + density_matching="Fixed ns-cell p_data/p_sim on parent objects across all artificial branches; DES table fixed, not refitted after matching.", + grids={}, + ) + maps, cells, fractions = [], [], [] + for grid in range(1, 5): + # Capture legacy unsquared weights on the same joined input arrays. + old = {} + original = calibration.metacal.get_variance_ivweights + names = dict(NOSHEAR="ns", **{"1P": "p1", "1M": "m1", "2P": "p2", "2M": "m2"}) + + def capture(data, sigma_eps, mask=None, legacy=False, branch="NOSHEAR"): + old[names[branch]] = original(data, sigma_eps, mask, True, branch)[2] + return original(data, sigma_eps, mask, legacy, branch) + + with patch.object( + calibration.metacal, "get_variance_ivweights", staticmethod(capture) + ): + gm, tiles = load(grid, "1z2z", "iv", exact=True) + own, _ = load(grid, "1z2z", "des", min_count=3) + for b in response.BRANCHES: + np.testing.assert_array_equal(gm.mask_dict[b], own.mask_dict[b]) + np.testing.assert_allclose(gm.R_shear, own.R_shear) + np.testing.assert_array_equal(es, own.W[0]) + np.testing.assert_array_equal(er, own.W[1]) + policies = dict( + unweighted={b: np.ones(len(gm.ns["g1"])) for b in response.BRANCHES}, + iv_squared=gm.branch_weights, + iv_unsquared=old, + des_v2=table_weights(gm, table), + des_self=table_weights(gm, own.W), + ) + ns = gm.mask_dict["ns"] + snr = gm.ns["flux"] / gm.ns["flux_err"] + size = gm.ns["T"] / gm.ns["Tpsf"] + index = response.cell_index(snr, es) * len(wd[0]) + response.cell_index( + size, er + ) + shape = wd.shape + count = np.bincount(index[ns], minlength=wd.size).reshape(shape) + assert np.all(count > 0), "Data occupancy cannot be matched in empty sim cells" + ps = count / count.sum() + factor = pd / ps + density = factor.ravel()[index] + before = {k: pack(gm, w) for k, w in policies.items()} + after = { + k: pack(gm, {b: w[b] * density for b in response.BRANCHES}) + for k, w in policies.items() + } + rg = (gm.R_shear[0, 0] + gm.R_shear[1, 1]) / 2 + rsum = np.bincount(index[ns], weights=rg, minlength=wd.size).reshape(shape) + rcell = rsum / count + outside = ( + (snr[ns] < es[0]) + | (snr[ns] > es[-1]) + | (size[ns] < er[0]) + | (size[ns] > er[-1]) + ) + assert not outside.any() + # Check exact occupancy match and direct cell-sum shear identity. + matched = np.bincount( + index[ns], weights=density[ns], minlength=wd.size + ).reshape(shape) + np.testing.assert_allclose(matched / matched.sum(), pd, atol=1e-13) + np.testing.assert_allclose( + np.sum(pd * rcell), after["unweighted"]["shear"]["mean_diagonal"] + ) + summaries = dict( + median_snr=float(np.median(snr[ns])), + median_size=float(np.median(size[ns])), + fraction_snr_below_30=float(np.mean(snr[ns] < 30)), + occupancy_total_variation=float(np.abs(ps - pd).sum() / 2), + density_factor_min=float(factor.min()), + density_factor_max=float(factor.max()), + density_effective_n=float( + density[ns].sum() ** 2 / np.sum(density[ns] ** 2) + ), + per_cell_R_min=float(rcell.min()), + per_cell_R_max=float(rcell.max()), + ) + result["grids"][str(grid)] = dict( + selected=len(ns), + tiles=len(np.unique(tiles)), + sigma_eps=gm._sigma_eps, + responses=before, + ratios=ratios(before), + occupancy_matched_responses=after, + occupancy_matched_ratios=ratios(after), + count=count.tolist(), + sim_fraction=ps.tolist(), + sim_over_data_fraction=(ps / pd).tolist(), + mean_R_cell=rcell.tolist(), + mean_R_snr=(rsum.sum(1) / count.sum(1)).tolist(), + mean_R_size=(rsum.sum(0) / count.sum(0)).tolist(), + summaries=summaries, + ) + maps.append(ps / pd) + cells.append(rcell) + fractions.append(ps) + print( + "GRID", grid, json.dumps(result["grids"][str(grid)]["ratios"]), flush=True + ) + print("MATCHED", grid, json.dumps(ratios(after)), flush=True) + del gm, own + (output / "results.json").write_text(json.dumps(result, indent=2)) + + fig, axes = plt.subplots(2, 4, figsize=(17, 8), constrained_layout=True) + for i, ratio in enumerate(maps): + ax = axes[0, i] + im = ax.pcolormesh( + es, er, np.log10(ratio).T, cmap="RdBu_r", vmin=-1.5, vmax=1.5 + ) + ax.set( + xscale="log", + yscale="log", + xlabel="SNR", + ylabel="T / T_PSF,reconv", + title=f"Grid {i + 1}: selected sim / data fraction", + ) + fig.colorbar(im, ax=axes[0], label="log10 occupancy fraction ratio") + cs, cr = np.sqrt(es[:-1] * es[1:]), np.sqrt(er[:-1] * er[1:]) + for j in (0, 1): + ax = axes[1, j] + coords = cs if j == 0 else cr + axis = 1 if j == 0 else 0 + ax.plot(coords, pd.sum(axis), color="black", lw=2, label="v2 W_COUNT") + for i, ps in enumerate(fractions): + ax.plot(coords, ps.sum(axis), label=f"Grid {i + 1}") + ax.set( + xscale="log", + xlabel="SNR" if j == 0 else "T / T_PSF,reconv", + ylabel="Fraction per log bin", + ) + ax.legend(fontsize=8) + ax = axes[1, j + 2] + for i, ps in enumerate(fractions): + ax.plot(coords, ps.sum(axis) / pd.sum(axis), label=f"Grid {i + 1}") + ax.axhline(1, color="black", ls="--") + ax.set( + xscale="log", + xlabel="SNR" if j == 0 else "T / T_PSF,reconv", + ylabel="Sim / data marginal fraction", + ) + fig.savefig(output / "occupancy.png", dpi=180) + plt.close(fig) + + fig, axes = plt.subplots(2, 4, figsize=(17, 8), constrained_layout=True) + for i, rc in enumerate(cells): + ax = axes[0, i] + im = ax.pcolormesh(es, er, rc.T, cmap="viridis", vmin=0.2, vmax=1.1) + ax.set( + xscale="log", + yscale="log", + xlabel="SNR", + ylabel="T / T_PSF,reconv", + title=f"Grid {i + 1}: unweighted cell mean R_g", + ) + for j, (coords, key) in enumerate(((cs, "mean_R_snr"), (cr, "mean_R_size"))): + axes[1, j].plot( + coords, result["grids"][str(i + 1)][key], label=f"Grid {i + 1}" + ) + fig.colorbar(im, ax=axes[0], label="Mean (R_g11 + R_g22) / 2") + for j in (0, 1): + axes[1, j].set( + xscale="log", + xlabel="SNR" if j == 0 else "T / T_PSF,reconv", + ylabel="Unweighted mean R_g", + ) + axes[1, j].legend() + for j, base in enumerate(("unweighted", "iv_squared")): + ax = axes[1, j + 2] + kind = "shear" if j == 0 else "total" + key = f"{kind}_des_v2_over_{base}" + for name, field, marker in ( + ("Original", "ratios", "o"), + ("Data occupancy", "occupancy_matched_ratios", "s"), + ): + ax.plot( + range(1, 5), + [result["grids"][str(i)][field][key] for i in range(1, 5)], + marker=marker, + label=name, + ) + ax.axhline( + 0.734 / 0.602 if j == 0 else (0.734 - 0.022) / (0.610 - 0.002), + color="black", + ls="--", + label="v2 data", + ) + ax.set( + xlabel="Grid", + ylabel="R_shear DES / unw" if j == 0 else "R_total DES / IV squared", + xticks=range(1, 5), + ) + ax.legend(fontsize=8) + fig.savefig(output / "per_cell_response.png", dpi=180) + plt.close(fig) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(p) + p.add_argument("--output", type=Path, required=True) + args = p.parse_args() + inputs.configure(args) + run(args.output) diff --git a/scripts/validation/response/summarize_response.py b/scripts/validation/response/summarize_response.py new file mode 100644 index 000000000..aba7e7d36 --- /dev/null +++ b/scripts/validation/response/summarize_response.py @@ -0,0 +1,43 @@ +"""Check all 20 case-d products against Martin's reference measurements.""" + +import argparse +import json +from pathlib import Path + + +def summarize(root, output): + output.mkdir(parents=True, exist_ok=True) + results = [ + json.loads((root / f"grid_{i}/validation.json").read_text()) + for i in range(1, 5) + ] + summary = dict( + products=sum(len(r["branches"]) for r in results), + max_abs_R_difference=max( + b["max_abs_R_difference"] for r in results for b in r["branches"].values() + ), + max_relative_W_difference=max( + b["max_relative_W_difference"] + for r in results + for b in r["branches"].values() + ), + max_paired_m_difference=max( + abs(v["new_paired"] - v["martin_paired"]) + for r in results + for v in r["m"].values() + ), + ) + assert summary["products"] == 20 + for key, value in summary.items(): + if key != "products": + assert value <= 1e-14, (key, value) + (output / "summary.json").write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + summarize(args.root, args.output) diff --git a/scripts/validation/response/summarize_sim_vs_data_response.py b/scripts/validation/response/summarize_sim_vs_data_response.py new file mode 100644 index 000000000..1f4606538 --- /dev/null +++ b/scripts/validation/response/summarize_sim_vs_data_response.py @@ -0,0 +1,47 @@ +"""Print direct-response diagnostics from sim_vs_data_response.py (allocated node).""" + +import json +import sys +from pathlib import Path + +import numpy as np + +root = Path(sys.argv[1]) +d = json.loads((root / "results.json").read_text()) +es, er = np.array(d["edges_snr"]), np.array(d["edges_size"]) +pd = np.array(d["data_count"], float) +pd /= pd.sum() +print("Data marginals: fractions in first/last five bins, SNR then size") +print( + pd.sum(1)[:5].sum(), pd.sum(1)[-5:].sum(), pd.sum(0)[:5].sum(), pd.sum(0)[-5:].sum() +) +for grid, g in d["grids"].items(): + print("\nGrid", grid, "N", g["selected"]) + print("policy R_shear11/22 R_sel11/22 means(shear,selection)") + for policy, r in g["responses"].items(): + print( + policy, + np.round(r["shear"]["diagonal"], 6), + np.round(r["selection"]["diagonal"], 6), + round(r["shear"]["mean_diagonal"], 6), + round(r["selection"]["mean_diagonal"], 6), + ) + print("summaries", g["summaries"]) + ps = np.array(g["sim_fraction"]) + print( + "Sim marginals first/last five bins", + ps.sum(1)[:5].sum(), + ps.sum(1)[-5:].sum(), + ps.sum(0)[:5].sum(), + ps.sum(0)[-5:].sum(), + ) + print("R_snr", np.round(g["mean_R_snr"], 3)) + print("R_size", np.round(g["mean_R_size"], 3)) + print( + "matched mean responses", + { + k: [round(v[z]["mean_diagonal"], 6) for z in ("shear", "selection")] + for k, v in g["occupancy_matched_responses"].items() + }, + ) +print("Bin5 edges", es[5], er[5], "last5", es[-6], er[-6]) diff --git a/scripts/validation/response/summarize_v2_table.py b/scripts/validation/response/summarize_v2_table.py new file mode 100644 index 000000000..53a84b0f9 --- /dev/null +++ b/scripts/validation/response/summarize_v2_table.py @@ -0,0 +1,38 @@ +"""Combine production residual m/c across independent simulation grids.""" + +import argparse +import json +from pathlib import Path + +import numpy as np + + +def summarize(root): + rows = {} + for grid in range(1, 5): + product = json.loads((root / f"production_{grid}.json").read_text()) + p = product["weights"]["w_des"] + rows[str(grid)] = dict( + value=[p[k] for k in ("m1", "c1", "m2", "c2")], + error=[p[k + "_err"] for k in ("m1", "c1", "m2", "c2")], + covariance=p["mc_covariance"], + deleted=p["mc_jackknife"], + labels=product["jackknife_tiles"], + clipping=json.loads((root / f"clipping_{grid}.json").read_text()), + ) + values = np.array([r["value"] for r in rows.values()]) + errors = np.array([r["error"] for r in rows.values()]) + weights = errors**-2 + mean = dict( + value=(np.sum(weights * values, axis=0) / weights.sum(axis=0)).tolist(), + error=np.sqrt(1 / weights.sum(axis=0)).tolist(), + ) + result = dict(order=["m1", "c1", "m2", "c2"], grids=rows, combined=mean) + (root / "summary.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(mean, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + summarize(parser.parse_args().root) diff --git a/scripts/validation/response/v2_data_table.py b/scripts/validation/response/v2_data_table.py new file mode 100644 index 000000000..52c2f43d1 --- /dev/null +++ b/scripts/validation/response/v2_data_table.py @@ -0,0 +1,93 @@ +"""Measure production residual m with the named v2 data table on allocated nodes.""" + +import argparse +import json +from pathlib import Path + +import inputs +import numpy as np +import yaml +from astropy.io import fits +from inputs import EXTERNAL + +from sp_validation import calibration, catalog_builders, image_sim_config, io +from sp_validation.image_sims import ImageSimMBias + + +def run(grid, output): + directory = output / f"grid_{grid}" + clipping = {} + for name in ("1z2z",) + EXTERNAL: + label = f"{name}_grid_{grid}" + source = inputs.BASE / f"grid_{grid}" / label + cfg = yaml.safe_load((source / "config_mask.yaml").read_text()) + cfg = image_sim_config.calibration_config( + cfg, ["w_des"], source="external", product=inputs.PRODUCT + ) + data = io.read_catalogue(source / "shape_catalog_comprehensive_ngmix.fits") + masks, _ = catalog_builders.get_masks_from_config(cfg, data, verbose=False) + pre = catalog_builders.Mask.from_list( + masks, label="combined", verbose=False + )._mask + gm = calibration.metacal_from_config(data, pre, cfg["metacal"]) + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + clipping[name] = {} + for branch in gm.branches: + d = getattr(gm, branch) + sel = gm.mask_dict[branch] + snr, size = ( + d["flux"][sel] / d["flux_err"][sel], + d["T"][sel] / d["Tpsf"][sel], + ) + es, er = gm.W[:2] + outside = (snr < es[0]) | (snr > es[-1]) | (size < er[0]) | (size > er[-1]) + parent_snr = d["flux"] / d["flux_err"] + parent_size = d["T"] / d["Tpsf"] + parent_out = ( + (parent_snr < es[0]) + | (parent_snr > es[-1]) + | (parent_size < er[0]) + | (parent_size > er[-1]) + ) + clipping[name][branch] = dict( + selected=len(snr), + outside=int(outside.sum()), + fraction=float(outside.mean()), + parent=len(parent_snr), + parent_outside=int(parent_out.sum()), + ) + path = directory / label / "shape_catalog_cut_ngmix.fits" + path.parent.mkdir(parents=True, exist_ok=True) + template = inputs.REFERENCE / f"grid_{grid}/des_d" / label / path.name + calibrated, raw, weights, *_ = calibration.get_calibrated_m_c(gm, False) + with fits.open(template) as hdus: + product = fits.HDUList([hdus[0].copy(), hdus[1].copy()]) + for i in range(2): + np.testing.assert_allclose(product[1].data[f"e{i + 1}_uncal"], raw[i]) + product[1].data[f"e{i + 1}"] = calibrated[i] + product[1].data["w_des"] = weights + product.writeto(path, overwrite=True) + calibration.save_response_product(path, gm, tiles) + print(grid, name, cfg, flush=True) + cfg = yaml.safe_load( + (inputs.BASE / f"grid_{grid}/results/m_bias_config.yaml").read_text() + ) + est = ImageSimMBias( + {**cfg, "grids_dir": str(directory), "w_cols": ["w_des"], "pair_match": False} + ) + est.load_catalogs(False) + prod = est.run(False) + print(json.dumps(prod, indent=2), flush=True) + (output / f"production_{grid}.json").write_text(json.dumps(prod, indent=2)) + (output / f"clipping_{grid}.json").write_text(json.dumps(clipping, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(parser) + parser.add_argument("--grid", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + inputs.configure(args) + args.output.mkdir(parents=True, exist_ok=True) + run(args.grid, args.output) diff --git a/scripts/validation/response/validate_response.py b/scripts/validation/response/validate_response.py new file mode 100644 index 000000000..bee967326 --- /dev/null +++ b/scripts/validation/response/validate_response.py @@ -0,0 +1,233 @@ +"""Validate DES case d on Martin's unblinded grids (compute node only). + +python scripts/validation/response/validate_response.py --grid 1 --output /path/to/results +The input shear comes from the grid manifest-derived m_bias_config.yaml. +No blinded catalogue is accepted by this script. +""" + +import argparse +import json +import resource +import subprocess +import time +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import inputs +import matplotlib.pyplot as plt +import numpy as np +import yaml +from astropy.io import fits + +from sp_validation import calibration, catalog_builders, io +from sp_validation.catalog import match_catalogs_radec +from sp_validation.image_sims import ImageSimMBias +from sp_validation.response_diagnostics import pair_products, plot_response + + +def run(grid, output): + start = time.monotonic() + base = inputs.BASE / f"grid_{grid}" + reference = inputs.REFERENCE / f"grid_{grid}/des_d" + output.mkdir(parents=True, exist_ok=True) + with open(base / "results/m_bias_config.yaml") as fh: + cfg = yaml.safe_load(fh) + g_in = cfg["shear_amplitude"] + products = {} + summary = { + "grid": grid, + "injected_shear": g_in, + "branches": {}, + "sp_validation_commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip(), + "weight_scheme": "des", + "W_policy": "self-fitted; refit on spatial deletion", + "des_min_count": 0, + "metacal_step": 0.01, + "grid_bins": 20, + "mean_subtraction": False, + "paired_match_radius_deg": cfg["match_radius_deg"], + } + for name in ("1z2z", "1p2z", "1m2z", "1z2p", "1z2m"): + branch_start = time.monotonic() + label = f"{name}_grid_{grid}" + directory = base / label + with open(directory / "config_mask.yaml") as fh: + mask_config = yaml.safe_load(fh) + data = io.read_catalogue(directory / "shape_catalog_comprehensive_ngmix.fits") + masks, _ = catalog_builders.get_masks_from_config( + mask_config, data, verbose=False + ) + pre = catalog_builders.Mask.from_list( + masks, label="combined", verbose=False + )._mask + cm = mask_config["metacal"].copy() + cm.pop("global_R_weight", None) + # Martin's reference uses unpooled cells; production pooling is tested + # separately with the independent data table. + cm.update(weight_scheme="des", w_col="w_des", des_num_bins=20, des_min_count=0) + gm = calibration.metacal_from_config(data, pre, cm) + shear, raw, w, selected, _, _ = calibration.get_calibrated_m_c(gm, False) + tiles = np.char.strip(np.asarray(data["TILE_ID"][pre]).astype(str)) + cut = data[pre][selected] + outdir = output / label + outdir.mkdir(exist_ok=True) + path = outdir / "shape_catalog_cut_ngmix.fits" + refpath = reference / label / path.name + with fits.open(refpath) as hdus: + np.testing.assert_allclose(raw[0], hdus[1].data["e1_uncal"], atol=1e-12) + np.testing.assert_allclose(raw[1], hdus[1].data["e2_uncal"], atol=1e-12) + refR = np.array( + [[hdus[0].header[f"R_{i}{j}"] for j in (1, 2)] for i in (1, 2)] + ) + refRJ = np.array( + [[hdus[0].header[f"RJ_{i}{j}"] for j in (1, 2)] for i in (1, 2)] + ) + w_diff = np.max(abs(w / hdus[1].data["w_des"] - 1)) + hdus[1].data["e1"] = shear[0] + hdus[1].data["e2"] = shear[1] + hdus[1].data["w_iv"] = gm.ns["w_iv"][selected] + hdus[1].data["w_des"] = w + # Keep only the table/header; write the new uncertainty product below. + fits.HDUList([hdus[0].copy(), hdus[1].copy()]).writeto(path, overwrite=True) + with fits.open(path, mode="update") as hdus: + for i in range(2): + for j in range(2): + for prefix, val in [ + ("R", gm.R), + ("R_G", gm.R_shear_global), + ("R_S", gm.R_selection), + ]: + hdus[0].header[ + f"{prefix}{'_' if prefix == 'R' else ''}{i + 1}{j + 1}" + ] = val[i, j] + calibration.save_response_product(path, gm, tiles) + products[name] = dict( + g=shear, + raw=raw, + w=w, + ra=cut["RA"], + dec=cut["Dec"], + ) + summary["branches"][name] = dict( + n=len(w), + R=gm.R.tolist(), + RJ=gm.R_joint.tolist(), + max_abs_R_difference=float(np.max(abs(gm.R - refR))), + max_abs_RJ_difference=float(np.max(abs(gm.R_joint - refRJ))), + max_relative_W_difference=float(w_diff), + branch_counts=gm.branch_counts, + branch_migration=gm.branch_migration, + branch_weight_sums={k: float(v) for k, v in gm.branch_weight_sums.items()}, + seconds=time.monotonic() - branch_start, + ) + if name == "1z2z": + plot_response(gm, tiles, cut["RA"], cut["Dec"], outdir) + pair = pair_products( + cut["RA"], cut["Dec"], w, gm.R_shear, min_sep=1, max_sep=120, nbins=8 + ) + np.savez(outdir / "pair_products.npz", **pair) + fig, axes = plt.subplots(1, 2, figsize=(10, 4)) + for k, comp in enumerate((0, 3)): + ratio = ( + pair["products"][:, comp, comp] + / pair["global_products"][comp, comp] + - 1 + ) + axes[0].plot(pair["theta"], ratio, "o-", label=("11", "22")[k]) + axes[0].axhline(0, color="k", lw=0.5) + axes[0].set( + xscale="log", + xlabel="Separation [arcmin]", + ylabel="Pair Rg product / global product − 1", + ) + axes[0].legend() + for i, j in ((0, 1), (0, 2), (3, 1), (3, 2)): + axes[1].plot( + pair["theta"], + pair["products"][:, i, j] - pair["global_products"][i, j], + "o-", + label=f"{i},{j}", + ) + axes[1].set( + xscale="log", + xlabel="Separation [arcmin]", + ylabel="Off-diagonal pair − global product", + ) + axes[1].legend() + fig.tight_layout() + fig.savefig(output / "pair_response.png", dpi=160) + plt.close(fig) + summary["stage1"] = dict( + note="Shear-only; selection/weight derivatives omitted. No total-response adequacy verdict.", + max_diagonal_fraction=float( + np.nanmax( + abs( + pair["products"][:, [0, 3], [0, 3]] + / pair["global_products"][[0, 3], [0, 3]] + - 1 + ) + ) + ), + ) + with open(reference / "results/m_bias_results.yaml") as fh: + martin = yaml.safe_load(fh)["weights"]["w_des"] + estimator = ImageSimMBias( + { + **cfg, + "grids_dir": str(output), + "num": grid, + "w_cols": ["w_des"], + "pair_match": False, + } + ) + estimator.load_catalogs(False) + production = estimator.run(False) + summary["production_m"] = production + summary["m"] = {} + for comp, (p, m) in enumerate((("1p2z", "1m2z"), ("1z2p", "1z2m"))): + plus, minus = products[p], products[m] + means = [np.average(x["g"][comp], weights=x["w"]) for x in (plus, minus)] + unpaired = (means[0] - means[1]) / (2 * g_in) - 1 + ip, im = match_catalogs_radec( + plus["ra"], + plus["dec"], + minus["ra"], + minus["dec"], + thresh_deg=cfg["match_radius_deg"], + ) + pairw = 0.5 * (plus["w"][ip] + minus["w"][im]) + paired = np.average( + (plus["g"][comp, ip] - minus["g"][comp, im]) / (2 * g_in) - 1, weights=pairw + ) + intersection = ( + np.average(plus["g"][comp, ip], weights=plus["w"][ip]) + - np.average(minus["g"][comp, im], weights=minus["w"][im]) + ) / (2 * g_in) - 1 + np.testing.assert_allclose(production[f"m{comp + 1}"], unpaired, atol=1e-14) + error = production[f"m{comp + 1}_err"] + summary["m"][f"m{comp + 1}"] = dict( + martin_paired=martin[f"m{comp + 1}"], + new_paired=float(paired), + new_unpaired=float(unpaired), + intersection_branch_weighted=float(intersection), + unpaired_refit_W_tile_jk_error=float(error), + ) + summary["seconds"] = time.monotonic() - start + summary["max_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + with open(output / "validation.json", "w") as fh: + json.dump(summary, fh, indent=2) + print(json.dumps(summary, indent=2), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + inputs.add_arguments(parser) + parser.add_argument("--grid", type=int, default=1) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + inputs.configure(args) + run(args.grid, args.output) diff --git a/scripts/validation/wdes_response/README.md b/scripts/validation/wdes_response/README.md new file mode 100644 index 000000000..ed2c61abd --- /dev/null +++ b/scripts/validation/wdes_response/README.md @@ -0,0 +1,21 @@ +# Response weighted like the shear mean (#395): reference scripts + +The image-sim tests of issue #395 that define case d (DES table W fitted once +on the no-shear branch, evaluated on each branch's own SNR and size). The +production code in `src/sp_validation/response.py` and `calibration.py` +reproduces it; these scripts are kept as the reference it was checked against. + +`diff_jk.py` and `table.py` use the old m-bias API +(`image_sims._jackknife_std`, `results["joint"]`) and do not run on the current +code. +Paths are hard-coded to candide. + +- `wdes_response.py`: recalibrate one grid's branches with the weightings iv, + des_c, des_d, iv_d, iv_box; writes one cut catalogue per weighting. +- `mbias_config.py`: m-bias config per weighting, from the workflow's config. +- `diff_jk.py`: tile-jackknife error of the m difference between weightings. +- `table.py`: grid table of m1, m2 and the differences. +- `job_wdes.sh`: SLURM array over the four grids. + +Outputs: `/n23data1/mkilbing/imsim_wdes_response/grid_*//`. +The reproduction scripts in `scripts/validation/response/` read these reference products; see that directory's README for commands. diff --git a/scripts/validation/wdes_response/diff_jk.py b/scripts/validation/wdes_response/diff_jk.py new file mode 100644 index 000000000..650db8aa9 --- /dev/null +++ b/scripts/validation/wdes_response/diff_jk.py @@ -0,0 +1,66 @@ +"""Tile-jackknife error of the difference in m between two response +weightings, e.g. iv_d - iv. Both share the same objects and shear mean, so +the error of the difference is much smaller than either error alone. + +Runs ImageSimMBias on both schemes' m_bias_config.yaml and records the +leave-one-tile-out samples of m that _jackknife_std receives. + +Usage: python diff_jk.py [...] +Pairs are (a, b): prints m_b - m_a per weight column, method and component, +and writes them to /diff__.yaml. +""" + +import os +import sys + +import numpy as np +import yaml + +from sp_validation import image_sims + + +def samples(out_root, scheme): + """Per-tile m samples: {(method, w_col, comp): array}.""" + with open(f"{out_root}/{scheme}/m_bias_config.yaml") as f: + cfg = yaml.safe_load(f) + mb = image_sims.ImageSimMBias(cfg) + mb.load_catalogs(verbose=False) + got = [] + orig = image_sims._jackknife_std + image_sims._jackknife_std = lambda x: (got.append(np.array(x)), orig(x))[1] + try: + out = {} + for method in ("default", "joint"): + got.clear() + for name_p, name_m, comp in mb.pairs: + res = mb._m_c_pair(name_p, name_m, comp, verbose=False, method=method) + # _jackknife_pair calls m then c for each w_col in order + for k, w_col in enumerate(mb.w_cols): + out[(method, w_col, comp + 1)] = ( + res[w_col]["m"], + got[-2 * len(mb.w_cols) + 2 * k], + ) + return out + finally: + image_sims._jackknife_std = orig + + +def main(): + out_root = sys.argv[1] + pairs = list(zip(sys.argv[2::2], sys.argv[3::2])) + for a, b in pairs: + sa, sb = samples(out_root, a), samples(out_root, b) + res = {} + for key in sa: + method, w_col, comp = key + d = sb[key][0] - sa[key][0] + err = image_sims._jackknife_std(sb[key][1] - sa[key][1]) + res.setdefault(method, {}).setdefault(w_col, {})[f"dm{comp}"] = float(d) + res[method][w_col][f"dm{comp}_err_jk"] = float(err) + print(f"{b} - {a} {method:7s} {w_col:6s} m{comp}: {d:+.5f} +/- {err:.5f}") + with open(os.path.join(out_root, f"diff_{b}_{a}.yaml"), "w") as f: + yaml.dump(res, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/wdes_response/job_wdes.sh b/scripts/validation/wdes_response/job_wdes.sh new file mode 100644 index 000000000..f778271c9 --- /dev/null +++ b/scripts/validation/wdes_response/job_wdes.sh @@ -0,0 +1,38 @@ +#!/usr/bin/bash +#SBATCH --output=/n23data1/mkilbing/imsim_wdes_response/job_%x_%A_%a.out +#SBATCH --partition=comp,pscomp +#SBATCH --job-name=wdes_resp +#SBATCH --array=1-4 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=2 +#SBATCH --mem=24G +#SBATCH --time=04:00:00 +#SBATCH --exclude=n09,n17,n23,n24,n25,n34,n35,n36 +set -euo pipefail +export SCR="/n23data1/mkilbing/shapepipe/tmp/${SLURM_JOB_NAME}_${SLURM_JOB_ID}" +mkdir -p "$SCR" +trap 'rm -rf "$SCR"' EXIT +export TMPDIR="$SCR" APPTAINER_TMPDIR="$SCR" SINGULARITY_TMPDIR="$SCR" + +G=$SLURM_ARRAY_TASK_ID +W=/n23data1/mkilbing/imsim_wdes_response +OUT=$W/grid_$G +SPV=/n17data/mkilbing/astro/repositories/github/sp_validation_spv +GRID=/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow/grid_$G +SIF=/n17data/cdaley/containers/sp_validation_im_sims.sif +mkdir -p $OUT + +run() { + apptainer exec --cleanenv --bind /n17data,/n23data1,/n09data,/home \ + --env TMPDIR=$SCR,MPLCONFIGDIR=$SCR,XDG_CACHE_HOME=$SCR,PYTHONPATH=$SPV/src,OMP_NUM_THREADS=1,HOME=$HOME,USER=$USER \ + $SIF "$@" +} + +run python $W/wdes_response.py $GRID $OUT +for S in iv iv_d iv_box des_c des_d; do + run python $W/mbias_config.py $G $S $OUT + cd $OUT/$S + run python $SPV/scripts/compute_m_bias_image_sims.py -c $OUT/$S/m_bias_config.yaml +done +run python $W/diff_jk.py $OUT iv iv_d iv iv_box des_c des_d +echo "exit code 0" diff --git a/scripts/validation/wdes_response/mbias_config.py b/scripts/validation/wdes_response/mbias_config.py new file mode 100644 index 000000000..48b61a39e --- /dev/null +++ b/scripts/validation/wdes_response/mbias_config.py @@ -0,0 +1,17 @@ +"""Write an m-bias config for one grid and response weighting, from the +workflow's own config with only the paths changed.""" + +import sys + +import yaml + +grid, scheme, out_root = sys.argv[1], sys.argv[2], sys.argv[3] +base = "/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow" +with open(f"{base}/grid_{grid}/results/m_bias_config.yaml") as f: + cfg = yaml.safe_load(f) +cfg["grids_dir"] = f"{out_root}/{scheme}" +cfg["results_dir"] = f"{out_root}/{scheme}/results" +cfg["output_path"] = f"{out_root}/{scheme}/results/m_bias_results.yaml" +cfg["provenance"]["response_weighting"] = f"{scheme} (wdes_response.py)" +with open(f"{out_root}/{scheme}/m_bias_config.yaml", "w") as f: + yaml.dump(cfg, f, default_flow_style=False) diff --git a/scripts/validation/wdes_response/table.py b/scripts/validation/wdes_response/table.py new file mode 100644 index 000000000..4a8be904a --- /dev/null +++ b/scripts/validation/wdes_response/table.py @@ -0,0 +1,65 @@ +import numpy as np +import yaml + +W = "/n23data1/mkilbing/imsim_wdes_response/grid_%d/%s/results/m_bias_results.yaml" +ORIG = "/n17data/mkilbing/astro/Runs/shapepipe/CFIS/v2.0/image_sims/new_workflow/grid_%d/results/m_bias_results.yaml" + + +def get(y, meth, w): + return y["weights"][w] if meth == "split" else y["joint"]["weights"][w] + + +# check iv reproduces original +for g in (1, 2, 3, 4): + a = yaml.safe_load(open(W % (g, "iv"))) + b = yaml.safe_load(open(ORIG % g)) + d = max( + abs(get(a, m, "w_iv")[k] - get(b, m, "w_iv")[k]) + for m in ("split", "joint") + for k in ("m1", "m2", "m1_err_jk", "m2_err_jk") + ) + print(f"iv check grid_{g}: max diff {d:.1e}") +rows = [ + ("w_iv, R w_iv (existing)", "iv", "w_iv"), + ("w_iv, R w_iv per branch (fit)", "iv_d", "w_iv"), + ("w_iv, R w_iv per branch (boxes)", "iv_box", "w_iv"), + ("w_des, R w_iv (existing)", "iv", "w_des"), + ("w_des, R w_des fixed (c)", "des_c", "w_des"), + ("w_des, R w_des per branch (d)", "des_d", "w_des"), +] +for meth in ("split", "joint"): + print(f"\n== {'R_shear + R_sel' if meth == 'split' else 'R_joint'}") + for lab, s, w in rows: + ys = [yaml.safe_load(open(W % (g, s))) for g in (1, 2, 3, 4)] + print(f" {lab}") + for k in ("m1", "m2"): + x = np.array([get(y, meth, w)[k] for y in ys]) + e = np.array([get(y, meth, w)[k + "_err_jk"] for y in ys]) + iv = 1 / e**2 + mu = (iv * x).sum() / iv.sum() + s_ = iv.sum() ** -0.5 + ch = ((x - mu) ** 2 * iv).sum() + per = " ".join(f"{v:+.4f}+/-{u:.4f}" for v, u in zip(x, e)) + print(f" {k} {per} mean {mu:+.4f} +/- {s_:.4f} chi2 {ch:.1f}/3") + +# differences with their own jackknife error (diff_jk.py) +D = "/n23data1/mkilbing/imsim_wdes_response/grid_%d/diff_%s_%s.yaml" +for b, a, w in ( + ("iv_d", "iv", "w_iv"), + ("iv_box", "iv", "w_iv"), + ("des_d", "des_c", "w_des"), +): + print(f"\n== {b} - {a}, weight {w}") + for meth in ("default", "joint"): + ys = [yaml.safe_load(open(D % (g, b, a)))[meth][w] for g in (1, 2, 3, 4)] + for k in ("dm1", "dm2"): + x = np.array([y[k] for y in ys]) + e = np.array([y[k + "_err_jk"] for y in ys]) + iv = 1 / e**2 + mu = (iv * x).sum() / iv.sum() + s_ = iv.sum() ** -0.5 + ch = ((x - mu) ** 2 * iv).sum() + per = " ".join(f"{v:+.4f}+/-{u:.4f}" for v, u in zip(x, e)) + print( + f" {meth:7s} {k} {per} mean {mu:+.4f} +/- {s_:.4f} chi2 {ch:.1f}/3" + ) diff --git a/scripts/validation/wdes_response/wdes_response.py b/scripts/validation/wdes_response/wdes_response.py new file mode 100644 index 000000000..f6497262b --- /dev/null +++ b/scripts/validation/wdes_response/wdes_response.py @@ -0,0 +1,384 @@ +"""Image-sim m-bias with the response weighted like the shear mean (#395). + +For each branch of one grid, rerun the calibration of +calibrate_comprehensive_cat.py in memory (same config, same code), then +recompute the response with three weightings and write one recalibrated +copy of the cut catalogue per weighting: + + iv : w_iv on every branch. Check: must reproduce the catalogue's R, RJ. + des_c : w_des = W(snr, size_ratio) from the no-shear quantities, the same + weight on every branch (no weight response). + des_d : w_des table W evaluated on each branch's own snr and size_ratio + (weight response included). The table itself is fixed: built once + from the no-shear sample, as in get_w_des. + iv_d : w_iv = 1/(2 sigma_eps^2 + C), C = G1_ERR + G2_ERR, re-evaluated on + each branch (weight response included). The catalogue keeps C only + for no-shear, so C_b = C_ns * exp(f(snr_b, size_b) - f(snr_ns, + size_ns)), with f a degree-2 polynomial fit of ln C_ns in + (ln snr, ln size). Each object keeps its own residual, so iv_d is + iv at no shear. Objects without a valid no-shear snr or size + (T_ns <= 0) take C_b = exp(f(snr_b, size_b)). A proxy: misses any + change of C at fixed (snr, size). + iv_box: as iv_d, but the change of C comes from the w_des boxes instead + of the fit: C_b = C_ns * Cbox(snr_b, size_b) / Cbox(snr_ns, + size_ns), Cbox = mean C of the no-shear selection per box. Values + outside the range go to the nearest edge box; a galaxy landing in + an empty box keeps C_ns. + +Each weighting gives R = R_shear + R_sel (split) and R_joint. Leave-one- +tile-out responses go in the R_JK HDU, as in the calibration script. + +Usage: python wdes_response.py +""" + +import os +import sys + +import numpy as np +from astropy.io import fits + +from sp_validation import calibration +from sp_validation import catalog_builders as sp_joint +from sp_validation.calibration import metacal + +BRANCHES = {"ns": "NOSHEAR", "p1": "1P", "m1": "1M", "p2": "2P", "m2": "2M"} +N_BINS = 20 +DEG_C = 2 + + +def poly_terms(x, y, deg=DEG_C): + return np.array([x**i * y**j for i in range(deg + 1) for j in range(deg + 1 - i)]).T + + +def fit_lnC(snr, size, C): + """Coefficients of the polynomial fit of ln C in (ln snr, ln size).""" + x, y = np.log(snr), np.log(size) + coef, *_ = np.linalg.lstsq(poly_terms(x, y), np.log(C), rcond=None) + resid = np.log(C) - poly_terms(x, y) @ coef + return coef, resid + + +def eval_lnC(snr, size, coef): + return poly_terms(np.log(snr), np.log(size)) @ coef + + +def w_table(snr, size_ratio, e1, e2, R11, R22, cm): + """Bin edges and cell values of w_des, as in calibration.get_w_des.""" + edges_s = calibration.create_bins( + snr, N_BINS, x_min=cm["gal_snr_min"], x_max=cm["gal_snr_max"] + ) + edges_r = calibration.create_bins( + size_ratio, N_BINS, x_min=cm["gal_rel_size_min"], x_max=cm["gal_rel_size_max"] + ) + i_s, i_r = cell_index(snr, size_ratio, edges_s, edges_r, like_pandas=True) + table = np.full((N_BINS, N_BINS), np.nan) + for i in range(N_BINS): + for j in range(N_BINS): + sel = (i_s == i) & (i_r == j) + if sel.sum() == 0: + continue + noise = 0.5 * (np.mean(e1[sel] ** 2) + np.mean(e2[sel] ** 2)) + resp = 0.5 * (np.mean(R11[sel]) + np.mean(R22[sel])) + table[i, j] = resp**2 / noise + return edges_s, edges_r, table + + +def cell_index(snr, size_ratio, edges_s, edges_r, like_pandas=False): + """Cell indices. like_pandas: as pd.cut(right=True) plus the code's rule + that values outside the range go to the last bin. Otherwise values + outside the range go to the nearest edge cell.""" + out = [] + for x, e in ((snr, edges_s), (size_ratio, edges_r)): + idx = np.searchsorted(e, x, side="left") - 1 + if like_pandas: + idx[(x <= e[0]) | (x > e[-1]) | ~np.isfinite(x)] = N_BINS - 1 + else: + idx = np.clip(idx, 0, N_BINS - 1) + out.append(idx) + return out + + +def eval_table(snr, size_ratio, edges_s, edges_r, table): + i_s, i_r = cell_index(snr, size_ratio, edges_s, edges_r) + w = table[i_s, i_r] + n_bad = int(np.sum(~np.isfinite(w))) + w[~np.isfinite(w)] = 0.0 + return w, n_bad + + +def responses(gm, wts, keep=None): + """Split R = R_shear + R_sel and joint R, weights per branch. + + wts[b] is a weight per object of the metacal input (gm.ns index), used + on branch b's selection. keep: bool per object, False drops it. + """ + h2 = 2 * gm._step + branch = {"p1": gm.p1, "m1": gm.m1, "p2": gm.p2, "m2": gm.m2} + sel = {} + for b, idx in gm.mask_dict.items(): + sel[b] = idx if keep is None else idx[keep[idx]] + + def mean(data, b, comp): + return np.average(data[comp][sel[b]], weights=wts[b][sel[b]]) + + ma = gm.mask_dict["ns"] + k_ns = np.ones(len(ma), bool) if keep is None else keep[ma] + R_shear = np.average(gm.R_shear[:, :, k_ns], axis=2, weights=wts["ns"][ma][k_ns]) + + R_sel = np.zeros((2, 2)) + R_joint = np.zeros((2, 2)) + for a, comp in enumerate(("g1", "g2")): + for b, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + R_sel[a, b] = (mean(gm.ns, p, comp) - mean(gm.ns, m, comp)) / h2 + R_joint[a, b] = (mean(branch[p], p, comp) - mean(branch[m], m, comp)) / h2 + return R_shear + R_sel, R_joint, R_shear, R_sel + + +def snr_size(dat_m, key): + snr = dat_m[f"NGMIX_FLUX_{key}"] / dat_m[f"NGMIX_FLUX_ERR_{key}"] + size = dat_m[f"NGMIX_T_{key}"] / dat_m[f"NGMIX_T_PSF_RECONV_{key}"] + return np.asarray(snr, float), np.asarray(size, float) + + +def run_branch(sim_dir, out_root): + os.chdir(sim_dir) + name = os.path.basename(sim_dir.rstrip("/")) + print(f"=== {name}", flush=True) + + obj = sp_joint.CalibrateCat() + config = obj.read_config_set_params("config_mask.yaml") + dat, dat_ext = obj.read_cat(load_into_memory=False) + masks, labels = sp_joint.get_masks_from_config(config, dat, dat_ext, verbose=False) + mask_combined = sp_joint.Mask.from_list(masks, label="combined", verbose=False) + cm = config["metacal"] + gm = metacal( + dat, + mask_combined._mask, + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + rel_size_min=cm["gal_rel_size_min"], + rel_size_max=cm["gal_rel_size_max"], + size_corr_ell=cm["gal_size_corr_ell"], + sigma_eps=cm["sigma_eps_prior"], + global_R_weight=cm["global_R_weight"], + verbose=False, + ) + _, g_uncorr, _, mask_metacal, _, _ = calibration.get_calibrated_m_c( + gm, additive_correction=cm.get("additive_correction", True) + ) + cat_gal = {} + sp_joint.compute_weights_gatti( + cat_gal, + g_uncorr, + gm, + dat, + mask_combined, + mask_metacal, + num_bins=N_BINS, + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + size_ratio_min=cm["gal_rel_size_min"], + size_ratio_max=cm["gal_rel_size_max"], + ) + + dat_m = dat[mask_combined._mask] + tiles = np.char.strip(np.asarray(dat["TILE_ID"]).astype(str)[mask_combined._mask]) + + # w_des table from the no-shear sample, checked against the catalogue's + edges_s, edges_r, table = w_table( + cat_gal["snr"], + cat_gal["size_ratio"], + g_uncorr[0], + g_uncorr[1], + cat_gal["R_g11"], + cat_gal["R_g22"], + cm, + ) + snr_ns, size_ns = snr_size(dat_m, "NOSHEAR") + w_c, bad_c = eval_table(snr_ns, size_ns, edges_s, edges_r, table) + ma = gm.mask_dict["ns"] + dev = np.max(np.abs(w_c[ma] / cat_gal["w_des"] - 1)) + print(f" w_des table check: max rel dev on S_ns = {dev:.2e}", flush=True) + if dev > 1e-6: + raise ValueError(f"{name}: w_des table does not reproduce w_des") + + w_d = {"ns": w_c} + for b, key in BRANCHES.items(): + if b == "ns": + continue + s, r = snr_size(dat_m, key) + w_d[b], n_bad = eval_table(s, r, edges_s, edges_r, table) + print(f" {b}: {n_bad} objects in empty w_des cells", flush=True) + print(f" c: {bad_c} objects in empty w_des cells", flush=True) + + # w_iv per branch: C from a fit on the no-shear selection + sig2 = 2 * cm["sigma_eps_prior"] ** 2 + C_ns = np.asarray( + dat_m["NGMIX_G1_ERR_NOSHEAR"] + dat_m["NGMIX_G2_ERR_NOSHEAR"], float + ) + w_iv_ns = 1 / (sig2 + C_ns) + dev = np.max(np.abs(w_iv_ns / gm.ns["w"] - 1)) + print(f" w_iv check: max rel dev = {dev:.2e}", flush=True) + if dev > 1e-12: + raise ValueError(f"{name}: w_iv from G_ERR does not reproduce gm.ns['w']") + coef, resid = fit_lnC(snr_ns[ma], size_ns[ma], C_ns[ma]) + print( + f" ln C fit (deg {DEG_C}) on S_ns: rms resid {resid.std():.4f}," + f" coef {np.array2string(coef, precision=4)}", + flush=True, + ) + with np.errstate(invalid="ignore"): + f_ns = eval_lnC(snr_ns, size_ns, coef) + no_ns = ~np.isfinite(f_ns) + w_iv_d = {"ns": gm.ns["w"]} + for b, key in BRANCHES.items(): + if b == "ns": + continue + s, r = snr_size(dat_m, key) + with np.errstate(invalid="ignore"): + f_b = eval_lnC(s, r, coef) + C_b = np.where(no_ns, np.exp(f_b), C_ns * np.exp(f_b - f_ns)) + w_iv_d[b] = 1 / (sig2 + C_b) + sel = gm.mask_dict[b] + print( + f" iv_d {b}: {int(no_ns[sel].sum())} objects on S_{b} without" + " valid no-shear snr/size, fit value used", + flush=True, + ) + if not np.all(np.isfinite(w_iv_d[b][sel])): + raise ValueError(f"{name}: non-finite iv_d weight on S_{b}") + ratio = w_iv_d[b][sel] / w_iv_ns[sel] - 1 + comp = "g1" if b in ("p1", "m1") else "g2" + rho = np.corrcoef(ratio, gm.ns[comp][sel])[0, 1] + print( + f" iv_d {b}: w/w_ns - 1 median {np.median(ratio):+.2e}" + f" rms {ratio.std():.2e}, corr with {comp}_ns {rho:+.3f}", + flush=True, + ) + + # w_iv per branch: change of C from the mean C per w_des box + i_s, i_r = cell_index(snr_ns[ma], size_ns[ma], edges_s, edges_r, like_pandas=True) + Cbox = np.full((N_BINS, N_BINS), np.nan) + for i in range(N_BINS): + for j in range(N_BINS): + t = (i_s == i) & (i_r == j) + if t.any(): + Cbox[i, j] = C_ns[ma][t].mean() + j_s, j_r = cell_index(snr_ns, size_ns, edges_s, edges_r) + Cbox_ns = Cbox[j_s, j_r] + w_iv_box = {"ns": gm.ns["w"]} + for b, key in BRANCHES.items(): + if b == "ns": + continue + s, r = snr_size(dat_m, key) + k_s, k_r = cell_index(s, r, edges_s, edges_r) + ratio_C = Cbox[k_s, k_r] / Cbox_ns + sel = gm.mask_dict[b] + n_empty = int(np.sum(~np.isfinite(ratio_C[sel]))) + n_moved = int(np.sum((k_s != j_s)[sel] | (k_r != j_r)[sel])) + ratio_C[~np.isfinite(ratio_C)] = 1.0 + w_iv_box[b] = 1 / (sig2 + C_ns * ratio_C) + ratio = w_iv_box[b][sel] / w_iv_ns[sel] - 1 + comp = "g1" if b in ("p1", "m1") else "g2" + rho = np.corrcoef(ratio, gm.ns[comp][sel])[0, 1] + print( + f" iv_box {b}: {n_moved} of {len(sel)} change box, {n_empty} in empty" + f" box; w/w_ns - 1 rms {ratio.std():.2e}, corr with {comp}_ns {rho:+.3f}", + flush=True, + ) + + schemes = { + "iv": {b: gm.ns["w"] for b in BRANCHES}, + "iv_d": w_iv_d, + "iv_box": w_iv_box, + "des_c": {b: w_c for b in BRANCHES}, + "des_d": w_d, + } + + # Check: iv reproduces the response of the catalogue on disk + cut_path = os.path.join(sim_dir, "shape_catalog_cut_ngmix.fits") + with fits.open(cut_path) as hdul: + hdr = hdul[0].header + R_disk = np.array([[hdr[f"R_{i}{j}"] for j in (1, 2)] for i in (1, 2)]) + RJ_disk = np.array([[hdr[f"RJ_{i}{j}"] for j in (1, 2)] for i in (1, 2)]) + e_uncal_disk = np.array([hdul[1].data["e1_uncal"], hdul[1].data["e2_uncal"]]) + R_iv, RJ_iv, _, _ = responses(gm, schemes["iv"]) + print( + f" iv check: |R - R_disk| = {np.max(np.abs(R_iv - R_disk)):.2e}," + f" |RJ - RJ_disk| = {np.max(np.abs(RJ_iv - RJ_disk)):.2e}", + flush=True, + ) + if np.max(np.abs(R_iv - R_disk)) > 1e-8 or np.max(np.abs(RJ_iv - RJ_disk)) > 1e-8: + raise ValueError(f"{name}: iv response does not reproduce the catalogue") + if np.max(np.abs(e_uncal_disk - g_uncorr)) > 1e-10: + raise ValueError(f"{name}: object order differs from the catalogue") + + labels_jk = np.unique(tiles) + for scheme, wts in schemes.items(): + R, RJ, R_sh, R_se = responses(gm, wts) + n_eff = { + b: float( + wts[b][gm.mask_dict[b]].sum() ** 2 + / (wts[b][gm.mask_dict[b]] ** 2).sum() + ) + for b in BRANCHES + } + print( + f" {scheme}: R_shear diag {R_sh[0, 0]:.4f} {R_sh[1, 1]:.4f}" + f" R_sel diag {R_se[0, 0]:+.4f} {R_se[1, 1]:+.4f}" + f" RJ diag {RJ[0, 0]:.4f} {RJ[1, 1]:.4f}" + f" N_eff(ns) {n_eff['ns']:.0f} of {len(ma)}", + flush=True, + ) + R_jk = np.empty((len(labels_jk), 2, 2)) + RJ_jk = np.empty((len(labels_jk), 2, 2)) + for k, t in enumerate(labels_jk): + R_jk[k], RJ_jk[k], _, _ = responses(gm, wts, keep=tiles != t) + + out_dir = os.path.join(out_root, scheme, name) + os.makedirs(out_dir, exist_ok=True) + with fits.open(cut_path) as hdul: + g = np.linalg.inv(R) @ g_uncorr + hdul[1].data["e1"] = g[0] + hdul[1].data["e2"] = g[1] + h0 = hdul[0].header + for i in (1, 2): + for j in (1, 2): + h0[f"R_{i}{j}"] = R[i - 1, j - 1] + h0[f"RJ_{i}{j}"] = RJ[i - 1, j - 1] + h0[f"R_G{i}{j}"] = R_sh[i - 1, j - 1] + h0[f"R_S{i}{j}"] = R_se[i - 1, j - 1] + h0["RWEIGHT"] = (scheme, "weighting of the response (wdes_response.py)") + cols = [ + fits.Column( + name="TILE_ID", + array=labels_jk, + format=f"{max(len(t) for t in labels_jk)}A", + ) + ] + for key, resp in (("R", R_jk), ("RJ", RJ_jk)): + for i in (1, 2): + for j in (1, 2): + cols.append( + fits.Column( + name=f"{key}_{i}{j}", + array=resp[:, i - 1, j - 1], + format="D", + ) + ) + jk = fits.BinTableHDU.from_columns(cols, name="R_JK") + hdus = fits.HDUList([hdul[0], hdul[1], jk]) + hdus.writeto( + os.path.join(out_dir, "shape_catalog_cut_ngmix.fits"), overwrite=True + ) + + +def main(): + grid_dir, out_root = sys.argv[1], sys.argv[2] + num = grid_dir.rstrip("/").split("_")[-1] + for b in ("1z2z", "1p2z", "1m2z", "1z2p", "1z2m"): + run_branch(os.path.join(grid_dir, f"{b}_grid_{num}"), out_root) + + +if __name__ == "__main__": + main() diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index b594b9b6d..9763fb149 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -8,6 +8,8 @@ """ +import warnings + import numpy as np import pandas as pd import statsmodels.api as sm @@ -16,10 +18,162 @@ from shear_psf_leakage import leakage, run_object from sp_validation import catalog as sp_cat +from sp_validation import response from sp_validation.io import read_catalogue from sp_validation.statistics import jackknif_weighted_average2 +def metacal_from_config(data, mask, cm, verbose=False): + """One calibration entry point; old release configs are explicitly legacy. + + Missing weight_scheme preserves their historical R/c/unsquared IV policy. + New production configurations must declare des or iv and matching w_col. + """ + modern = "weight_scheme" in cm + return metacal( + data, + mask, + snr_min=cm["gal_snr_min"], + snr_max=cm["gal_snr_max"], + rel_size_min=cm["gal_rel_size_min"], + rel_size_max=cm["gal_rel_size_max"], + size_corr_ell=cm["gal_size_corr_ell"], + sigma_eps=cm["sigma_eps_prior"], + global_R_weight=None if modern else cm.get("global_R_weight"), + weight_scheme=cm.get("weight_scheme"), + legacy_response=not modern, + des_num_bins=cm.get("des_num_bins", 20), + des_min_count=cm.get("des_min_count", 3), + des_weight_product=cm.get("des_weight_product"), + w_col=cm.get("w_col"), + verbose=verbose, + ) + + +def load_des_weight_product(product): + """Read a named, independent calibration product, never refitting its W. + + Use modern response products, whose headers declare the coordinate + definitions. Historical released weights alone do not specify such a table. + The product's edges and pooling rule are authoritative, not the sim cuts. + """ + if ( + not isinstance(product, dict) + or not product.get("name") + or not product.get("path") + ): + raise ValueError("des_weight_product needs a name and a FITS path") + with fits.open(product["path"]) as hdus: + hdr = hdus[0].header + if ( + hdr.get("WGHTSCH") != "des" + or hdr.get("WSNRCOL") != "FLUX/FLUX_ERR" + or hdr.get("WSIZDEF") != "T/T_PSF_RECONV" + ): + raise ValueError( + "Incompatible DES product: regenerate with coordinate metadata" + ) + es, er = np.array(hdus["W_EDGES"].data, dtype=float) + w = np.array(hdus["W_DES"].data, dtype=float) + count = np.array(hdus["W_COUNT"].data) + if ( + w.shape != (len(es) - 1, len(er) - 1) + or count.shape != w.shape + or not np.all(np.diff(es) > 0) + or not np.all(np.diff(er) > 0) + or not np.isfinite(w).all() + or np.any(w < 0) + or not np.any(w > 0) + ): + raise ValueError("Invalid DES table or edges in calibration product") + return (es, er, w, count), hdr["DESNMIN"] + + +def save_response_product(path, gm, groups=None): + """Attach matrix definitions, fitted W, and spatial response/c covariance. + + groups are aligned to the preselected parent, never only the ns selection. + """ + if gm._legacy_response: + with fits.open(path, mode="update") as hdus: + hdus[0].header["RLEGACY"] = (True, "Historical R/c and unsquared IV policy") + return + with fits.open(path, mode="update") as hdus: + hdr = hdus[0].header + hdr["MEANSUB"] = getattr(gm, "subtract_mean_shear", False) + hdr["WGHTSCH"] = gm.weight_scheme + hdr["RAPPLIED"] = True + hdr["MCSTEP"] = gm._step + hdr["WPOLICY"] = ( + "fixed-external" + if gm._des_weight_product is not None + else "refit" + if gm.weight_scheme == "des" + else "not applicable" + ) + hdr["IVPOLICY"] = "branch variances" + hdr["CMUNITS"] = "raw ellipticity" + for i in range(2): + hdr[f"CE_{i + 1}"] = gm.c_e[i] + hdr[f"CG_{i + 1}"] = gm.c_g[i] + for j in range(2): + hdr[f"RJ_{i + 1}{j + 1}"] = gm.R_joint[i, j] + for b in response.BRANCHES: + hdr[f"N_{b}"] = gm.branch_counts[b] + hdr[f"SW_{b}"] = gm.branch_weight_sums[b] + if hasattr(gm, "branch_migration"): + hdr[f"MIG_{b}"] = gm.branch_migration[b] + if hasattr(gm, "W"): + hdr["DESNMIN"] = (gm._des_min_count, "Minimum DES training count") + hdr["DESMERGE"] = "concentric square; pooled moments" + hdr["WSNRCOL"] = "FLUX/FLUX_ERR" + hdr["WSIZDEF"] = "T/T_PSF_RECONV" + hdr["WSOURCE"] = "external" if gm._des_weight_product else "self" + if gm._des_weight_product: + hdr["WPRODUCT"] = gm._des_weight_product["name"] + hdr["WPRODPTH"] = gm._des_weight_product["path"] + es, er, w, count = gm.W + hdus.append(fits.ImageHDU(w, name="W_DES")) + hdus.append(fits.ImageHDU(count, name="W_COUNT")) + hdus.append(fits.ImageHDU(np.array([es, er]), name="W_EDGES")) + if groups is not None: + labels, estimates, cov = gm.response_jackknife( + groups, refit_w=gm.weight_scheme == "des" + ) + labels = np.asarray(labels).astype(str) + columns = [ + fits.Column( + name="TILE_ID", array=labels, format=f"{max(map(len, labels))}A" + ) + ] + for key, prefix in [ + ("total", "R"), + ("joint", "RJ"), + ("shear", "RG"), + ("selection", "RS"), + ]: + for i in range(2): + for j in range(2): + columns.append( + fits.Column( + name=f"{prefix}_{i + 1}{j + 1}", + array=estimates[key][:, i, j], + format="D", + ) + ) + for key in ("c_e", "c_g"): + for i in range(2): + columns.append( + fits.Column( + name=f"{key}{i + 1}", array=estimates[key][:, i], format="D" + ) + ) + hdus.append(fits.BinTableHDU.from_columns(columns, name="R_JK")) + cov_hdu = fits.ImageHDU(cov, name="R_COV") + cov_hdu.header["ORDER"] = "shear,selection,total,joint,c_e,c_g; row-major" + hdus.append(cov_hdu) + + def get_calibrated_quantities(gal_metacal): """Get Calibrated Quantities. @@ -98,11 +252,18 @@ def get_calibrated_m_c(gal_metacal, additive_correction=True): c_err = np.zeros(2) for comp in (0, 1): - c[comp] = np.mean(g_uncorr[comp]) + if getattr(gal_metacal, "_legacy_response", False): + c[comp] = np.mean(g_uncorr[comp]) + c_err[comp] = np.std(g_uncorr[comp]) + else: + c[comp] = np.average(g_uncorr[comp], weights=w) + c_err[comp] = ( + np.sqrt(np.sum(w**2 * (g_uncorr[comp] - c[comp]) ** 2)) / w.sum() + ) - # MKDEBUG TODO: Use std of mean instead, - # which is consistent with jackknife - c_err[comp] = np.std(g_uncorr[comp]) + gal_metacal.subtract_mean_shear = additive_correction + gal_metacal.c_e = c + gal_metacal.c_g = np.linalg.solve(gal_metacal.R, c) # Shear estimate corrected for additive bias g_corr_mc = np.copy(g_corr) @@ -176,7 +337,6 @@ def cut_to_bins(df, key, num_bins, type="log", x_min=None, x_max=None): def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="weights"): - cat_gal["e1_uncal"] = g_uncorr[0] cat_gal["e2_uncal"] = g_uncorr[1] cat_gal["R_g11"] = gal_metacal.R11 @@ -261,46 +421,33 @@ def get_w_des( DES weights """ - df_gal = build_df(cat_gal) - - # Create logarithmic bins in size and SNR - cut_to_bins( - df_gal, - "snr", - num_bins, - type="log", - x_min=snr_min, - x_max=snr_max, - ) - cut_to_bins( - df_gal, - "size_ratio", - num_bins, - type="log", - x_min=size_ratio_min, - x_max=size_ratio_max, + table = fit_w_des( + cat_gal, num_bins, snr_min, snr_max, size_ratio_min, size_ratio_max ) + return response.evaluate_w(table, cat_gal["snr"], cat_gal["size_ratio"]) - # Compute shape noise and the shear response in each bin - for i in range(num_bins): - for j in range(num_bins): - bin_mask = (df_gal["snr_log_bins"] == i) & ( - df_gal["size_ratio_log_bins"] == j - ) - ngal = np.sum(bin_mask) - if ngal == 0: - print(f"Zero galaxies in snr/size_ratio bin ({i},{j})") - shape_noise = 0.5 * ( - np.sum(df_gal[bin_mask]["e1_uncal"] ** 2) / ngal - + np.sum(df_gal[bin_mask]["e2_uncal"] ** 2) / ngal - ) - response = 0.5 * ( - np.average(df_gal[bin_mask]["R_g11"]) - + np.average(df_gal[bin_mask]["R_g22"]) - ) - df_gal.loc[bin_mask, "w_des"] = response**2 / shape_noise - return np.array(df_gal["w_des"]) +def fit_w_des( + cat_gal, + num_bins=20, + snr_min=None, + snr_max=None, + size_ratio_min=None, + size_ratio_max=None, + min_count=0, +): + """Fit once on raw no-shear shapes; evaluate via response.evaluate_w.""" + return response.fit_w( + np.asarray(cat_gal["snr"]), + np.asarray(cat_gal["size_ratio"]), + np.array([cat_gal["e1_uncal"], cat_gal["e2_uncal"]]), + 0.5 * (np.asarray(cat_gal["R_g11"]) + np.asarray(cat_gal["R_g22"])), + create_bins(cat_gal["snr"], num_bins, x_min=snr_min, x_max=snr_max), + create_bins( + cat_gal["size_ratio"], num_bins, x_min=size_ratio_min, x_max=size_ratio_max + ), + min_count=min_count, + ) def get_alpha_leakage_per_object(cat_gal, num_bins, weight_type="des"): @@ -522,7 +669,6 @@ def get_quantities_binned( which=["response", "number", "leakage"], verbose=True, ): - if verbose: print("Compute binned quantities") @@ -627,6 +773,23 @@ def get_calibrate_e_from_cat(path_cat_gal, weight_type="des", verbose=False): ] ) + stored_scheme = hdu_gal[0].header.get("WGHTSCH") + if stored_scheme != weight_type: + hdu_gal.close() + raise ValueError( + "Selection response cannot be reused with unknown/mismatched " + "weights; recalibrate from the parent five-branch catalogue" + ) + hdr = hdu_gal[0].header + weights = cat_gal["w_" + weight_type] + if hdr.get("N_ns") != len(cat_gal) or not np.isclose( + hdr.get("SW_ns", np.nan), weights.sum(), rtol=1e-12 + ): + hdu_gal.close() + raise ValueError( + "Stored selection response does not describe this " + "population/weights; use the parent five-branch catalogue" + ) if verbose: print("R_select\n", R_select) @@ -640,7 +803,9 @@ def get_calibrate_e_from_cat(path_cat_gal, weight_type="des", verbose=False): R_shear = np.zeros((2, 2)) for iidx in range(2): for jidx in range(2): - R_shear[iidx, jidx] = np.mean(cat_gal[f"R_g{iidx + 1}{jidx + 1}"]) + R_shear[iidx, jidx] = np.average( + cat_gal[f"R_g{iidx + 1}{jidx + 1}"], weights=cat_gal["w_" + weight_type] + ) if verbose: print("R_shear\n", R_shear) @@ -653,9 +818,11 @@ def get_calibrate_e_from_cat(path_cat_gal, weight_type="des", verbose=False): c_corr = np.linalg.inv(R) @ c g_cal = np.linalg.inv(R) @ g - for comp in (0, 1): - g_cal[comp] -= c_corr[comp] + if hdu_gal[0].header.get("MEANSUB", True): + for comp in (0, 1): + g_cal[comp] -= c_corr[comp] + hdu_gal.close() return g_cal[0], g_cal[1] @@ -741,8 +908,38 @@ def __init__( global_R_weight=None, sigma_eps=0.34, verbose=False, + weight_scheme=None, + legacy_response=False, + des_num_bins=20, + des_min_count=3, + des_weight_product=None, + w_col=None, ): - + if weight_scheme is not None and global_R_weight is not None: + raise ValueError("Use weight_scheme, not both response weight knobs") + if weight_scheme is None and not legacy_response: + if global_R_weight == "w": + warnings.warn( + "global_R_weight=w is deprecated; use weight_scheme=iv", + DeprecationWarning, + stacklevel=2, + ) + weight_scheme = "iv" + elif global_R_weight not in (None, "None"): + raise ValueError("Use weight_scheme=des or iv") + else: + legacy_response = True # Historical unweighted diagnostic API. + self.weight_scheme = weight_scheme + self._legacy_response = legacy_response + self._des_num_bins = des_num_bins + self._des_min_count = des_min_count + self._des_weight_product = des_weight_product + if des_weight_product is not None: + if weight_scheme != "des" or legacy_response: + raise ValueError("des_weight_product requires modern weight_scheme=des") + self.W, self._des_min_count = load_des_weight_product(des_weight_product) + if w_col is not None: + response.validate_weight_scheme(weight_scheme, w_col) self._masking_type = masking_type self._step = step @@ -819,7 +1016,20 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): masked_data, self._sigma_eps, mask=None, + legacy=self._legacy_response, ) + ns["w_iv"] = ns["w"] + if self.weight_scheme == "iv" and not self._legacy_response: + for name, branch in zip(("1M", "1P", "2M", "2P"), (m1, p1, m2, p2)): + required = [f"NGMIX_G{i}_ERR_{name}" for i in (1, 2)] + if not all(key in masked_data.dtype.names for key in required): + raise ValueError( + "IV response needs per-branch G1_ERR/G2_ERR; regenerate " + "the comprehensive catalogue from the tile final catalogues" + ) + branch["w_iv"] = self.get_variance_ivweights( + masked_data, self._sigma_eps, branch=name + )[2] self._n_input = len(masked_data) self._n_after_gal_mask = len(dict_tmp["flag"]) @@ -833,7 +1043,9 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): return m1, p1, m2, p2, ns @staticmethod - def get_variance_ivweights(data, sigma_eps, mask=None): + def get_variance_ivweights( + data, sigma_eps, mask=None, legacy=False, branch="NOSHEAR" + ): """Get Variance IVWEIGHTS. Compute variance and inverse-variance weights. @@ -858,8 +1070,11 @@ def get_variance_ivweights(data, sigma_eps, mask=None): weight """ - C11 = data["NGMIX_G1_ERR_NOSHEAR"] - C22 = data["NGMIX_G2_ERR_NOSHEAR"] + C11 = data[f"NGMIX_G1_ERR_{branch}"] + C22 = data[f"NGMIX_G2_ERR_{branch}"] + if not legacy: + C11 = C11**2 + C22 = C22**2 if mask is not None: C11 = C11[mask] C22 = C22[mask] @@ -883,11 +1098,173 @@ def _compute_calibration(self): raise ValueError(f"Invalid masking type '{self._masking_type}'") self._shear_response() - self._selection_response() - self._total_response() + if self._legacy_response: + self._selection_response() + self._total_response() + else: + self._prepare_weights() + result = self.response_estimates() + self.R_shear_global = result["shear"] + self.R_selection = result["selection"] + self.R = result["total"] + self.R_joint = result["joint"] + self.c_e, self.c_g = result["c_e"], result["c_g"] # self._shear_response_std(stat_operator=lambda x: # jackknif_weighted_average(x, np.ones_like(x))) + def _prepare_weights(self): + """Case d: fit W once, evaluate each branch's own SNR/size.""" + ma = self.mask_dict["ns"] + self.branches = { + b: (getattr(self, b)["g1"], getattr(self, b)["g2"]) + for b in response.BRANCHES + } + self.weight_metadata = dict( + scheme=self.weight_scheme, + step=self._step, + estimator="split", + W_policy="fixed", + des_min_count=self._des_min_count, + iv_policy="branch measurement variances", + c_units="c_e raw; c_g calibrated", + ) + if self.weight_scheme == "iv": + self.branch_weights = { + b: getattr(self, b)["w_iv"] for b in response.BRANCHES + } + else: + snr = self.ns["flux"] / self.ns["flux_err"] + size = self.ns["T"] / self.ns["Tpsf"] + if self._des_weight_product is None: + self.W = fit_w_des( + dict( + snr=snr[ma], + size_ratio=size[ma], + e1_uncal=self.ns["g1"][ma], + e2_uncal=self.ns["g2"][ma], + R_g11=self.R11, + R_g22=self.R22, + ), + self._des_num_bins, + self._snr_min, + self._snr_max, + self._rel_size_min, + self._rel_size_max, + min_count=self._des_min_count, + ) + ns_cell = ( + response.cell_index(snr, self.W[0]), + response.cell_index(size, self.W[1]), + ) + self.branch_weights = {} + self.branch_migration = {} + for b in response.BRANCHES: + d = getattr(self, b) + s, r = d["flux"] / d["flux_err"], d["T"] / d["Tpsf"] + self.branch_weights[b] = response.evaluate_w(self.W, s, r) + selected = self.mask_dict[b] + moved = (response.cell_index(s, self.W[0]) != ns_cell[0]) | ( + response.cell_index(r, self.W[1]) != ns_cell[1] + ) + self.branch_migration[b] = float(np.mean(moved[selected])) + self.ns["w_des"] = self.branch_weights["ns"] + self.ns["w"] = self.branch_weights["ns"] + self.branch_counts = {b: len(self.mask_dict[b]) for b in response.BRANCHES} + self.branch_weight_sums = { + b: self.branch_weights[b][self.mask_dict[b]].sum() + for b in response.BRANCHES + } + + def response_estimates(self, keep=None): + return response.responses( + self.branches, + self.mask_dict, + self.branch_weights, + self.R_shear, + self._step, + keep, + ) + + def refit_weights(self, keep): + """Refit DES W on retained S_ns, evaluate it on every parent branch. + + Edges and cuts stay fixed; cell moments and membership of the training + sample change. Artificial branches never train their own table. + """ + if self.weight_scheme != "des" or self._des_weight_product is not None: + return self.branch_weights + idx = self.mask_dict["ns"] + retained = keep[idx] + selected = idx[retained] + snr = self.ns["flux"] / self.ns["flux_err"] + size = self.ns["T"] / self.ns["Tpsf"] + table = response.fit_w( + snr[selected], + size[selected], + np.asarray(self.branches["ns"])[:, selected], + 0.5 * (self.R11[retained] + self.R22[retained]), + self.W[0], + self.W[1], + min_count=self._des_min_count, + ) + return { + b: response.evaluate_w( + table, + getattr(self, b)["flux"] / getattr(self, b)["flux_err"], + getattr(self, b)["T"] / getattr(self, b)["Tpsf"], + ) + for b in response.BRANCHES + } + + def response_jackknife(self, groups, refit_w=False, *, reference=False): + """Delete spatial groups, optionally refitting W from retained cell sums. + + reference=True keeps the original K full-parent rescans for validation. + Both paths use the full-sample edges and retained-count pooling rule. + """ + if refit_w and self.weight_scheme == "des" and self._des_weight_product is None: + if not reference: + from sp_validation.des_jackknife import accumulate + + return accumulate(self, groups).jackknife() + groups = np.asarray(groups) + if len(groups) != len(self.ns["g1"]): + raise ValueError("Spatial groups must align to the common parent rows") + labels = np.unique(groups) + if len(labels) < 2: + raise ValueError( + "Response jackknife requires at least two spatial groups" + ) + samples = [] + for label in labels: + keep = groups != label + weights = self.refit_weights(keep) + samples.append( + response.responses( + self.branches, + self.mask_dict, + weights, + self.R_shear, + self._step, + keep, + ) + ) + estimates = {key: np.array([s[key] for s in samples]) for key in samples[0]} + vectors = np.concatenate( + [x.reshape(len(labels), -1) for x in estimates.values()], axis=1 + ) + delta = vectors - vectors.mean(axis=0) + cov = (len(labels) - 1) / len(labels) * delta.T @ delta + return labels, estimates, cov + return response.jackknife( + self.branches, + self.mask_dict, + self.branch_weights, + self.R_shear, + self._step, + np.asarray(groups), + ) + def add_cuts(self, snr_min=10, snr_max=500, rel_size_min=0.5, rel_size_max=3.0): """Add Cuts. @@ -926,7 +1303,7 @@ def _masking_gal(self): [self.ns, self.m1, self.p1, self.m2, self.p2], ["ns", "m1", "p1", "m2", "p2"], ): - Tr_tmp = data["T"] + Tr_tmp = data["T"] if self._legacy_response else data["T"].copy() if self._size_corr_ell: Tr_tmp *= (1 - (data["g1"] ** 2 + data["g2"] ** 2)) / ( 1 + (data["g1"] ** 2 + data["g2"] ** 2) @@ -1099,7 +1476,6 @@ def _total_response(self): def mask_gal_size( T, Tpsf, rel_size_min, rel_size_max, size_corr_ell=False, g1=None, g2=None ): - Tr_tmp = T if size_corr_ell: Tr_tmp *= (1 - g1**2 + g2**2) / (1 + g1**2 + g2**2) @@ -1110,7 +1486,6 @@ def mask_gal_size( def mask_gal_SNR(SNR, snr_min, snr_max): - mask = (SNR > snr_min) & (SNR < snr_max) return mask diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index 69d1b36e2..9b9c1378e 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -414,7 +414,7 @@ def write_shape_catalog( signal-to-noise ratio, default is `None` g : np.ndarray, optional calibrated reduced shear estimate components, corrected for - multiplicative and additive bias, g = R^-1 g_uncal - c; + multiplicative and additive bias, g = R^-1 (g_uncal - c_e); expected type is arrays(2, ngal) of float; default is ``None`` (no calibrated shears written) g1_uncal, g2_uncal : np.ndarray, optional @@ -432,9 +432,9 @@ def write_shape_catalog( R_select : 2x2 matrix of float, optional Global selection response matrix, default is ``None`` c : array(2) of float, optional, default is ``None`` - additive shear bias + additive bias c_e in raw ellipticity units (c_g = R^-1 c_e) c_err : array(2) of float, optional, default is ``None`` - error of c + error of the raw weighted mean c_e; historical products may store scatter alpha_leakage : float, optional Mean scale-dependent PSF leakage, default is ``None`` sigma_epsilon: float, optional @@ -603,7 +603,7 @@ def write_shape_catalog( primary_header["c1_err"] = (c_err[0], "Standard deviation of c_1") primary_header["c2_err"] = (c_err[1], "Standard deviation of c_2") - primary_header["w"] = "DES weight" + primary_header["w"] = f"{w_type} weight" if sigma_epsilon is not None: primary_header["sig_eps"] = (sigma_epsilon, "Shape noise RMS") diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index 630cdc610..7f34e11e9 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -1085,14 +1085,18 @@ def compute_weights_gatti( purpose="weights", ) - cat_gal["w_des"] = calibration.get_w_des( - cat_gal, - num_bins, - snr_min=snr_min, - snr_max=snr_max, - size_ratio_min=size_ratio_min, - size_ratio_max=size_ratio_max, - ) + if gal_metacal.weight_scheme == "des": + cat_gal["w_des"] = gal_metacal.ns["w_des"][mask_metacal] + else: + cat_gal["w_des"] = calibration.get_w_des( + cat_gal, + num_bins, + snr_min=snr_min, + snr_max=snr_max, + size_ratio_min=size_ratio_min, + size_ratio_max=size_ratio_max, + ) + cat_gal["w_iv"] = gal_metacal.ns["w_iv"][mask_metacal] def compute_PSF_leakage( @@ -1102,17 +1106,17 @@ def compute_PSF_leakage( mask_combined, mask_metacal, num_bins=20, + weight_type="des", ): - """Compute PSF Leakage.""" + """Compute PSF leakage using the selected calibration weight.""" cat_gal["e1"] = g_corr_mc[0] cat_gal["e2"] = g_corr_mc[1] cat_gal["e1_PSF"] = sp_cat.get_col(dat, "e1_PSF", mask_combined._mask, mask_metacal) cat_gal["e2_PSF"] = sp_cat.get_col(dat, "e2_PSF", mask_combined._mask, mask_metacal) - weight_type = "des" key = f"w_{weight_type}" if key not in cat_gal: - raise KeyError("Key '{key}' not found in cat_gal") + raise KeyError(f"Key '{key}' not found in cat_gal") try: alpha_1, alpha_2 = calibration.get_alpha_leakage_per_object( diff --git a/src/sp_validation/cosmo_val/catalog_characterization.py b/src/sp_validation/cosmo_val/catalog_characterization.py index 556ee9f79..d122ffaf6 100644 --- a/src/sp_validation/cosmo_val/catalog_characterization.py +++ b/src/sp_validation/cosmo_val/catalog_characterization.py @@ -416,10 +416,10 @@ def calculate_additive_bias(self): self._c2 = {} for ver in self.versions: self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] with self.results[ver].temporarily_read_data(): e1, e2, w = self._read_shear_cols(ver, "e1_col", "e2_col", "w_col") - self._c1[ver], self._c2[ver] = additive_bias(e1, e2, w, R) + R1, R2 = self._component_responses(ver) + self._c1[ver], self._c2[ver] = additive_bias(e1 / R1, e2 / R2, w, 1) self.print_done("Finished additive bias calculation.") @property diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index a0beaf73a..bf7a155db 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -18,6 +18,7 @@ find_conservative_scale_cut_key, ) from ..statistics import chi2_and_pte +from ..survey import additive_bias from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin from .cosebis import CosebisMixin @@ -675,24 +676,42 @@ def _read_shear_cols(self, ver, *keys): ) return cols[0] if len(cols) == 1 else cols - def _calibrated_g(self, ver): + def _calibrated_g(self, ver, catalog=None): """Calibrated shear components ``(g1, g2)`` for a catalog version. Applies additive-bias subtraction and the multiplicative response: - ``g = (e − c) / R``. For DES the response is the catalog-averaged + ``g = e / R − c_g``. Stored c is in calibrated shear units. For DES the response is the catalog-averaged per-component ``R11``/``R22`` (column names in the config); for every other version it is the scalar ``R`` from the config. - Must be called inside a ``self.results[ver].temporarily_read_data()`` - context, since it reads ``dat_shear`` columns. + Without ``catalog``, call inside ``temporarily_read_data()``. With a + catalogue, calculate the same weighted mean directly from its full + population before any tomographic selection. + """ + if catalog is None: + e1, e2 = self._read_shear_cols(ver, "e1_col", "e2_col") + R1, R2 = self._component_responses(ver) + c1, c2 = self.c1[ver], self.c2[ver] + else: + shear = self.cc[ver]["shear"] + e1, e2 = catalog[shear["e1_col"]], catalog[shear["e2_col"]] + R1, R2 = self._component_responses(ver, catalog) + c1, c2 = additive_bias(e1 / R1, e2 / R2, catalog[shear["w_col"]], 1) + return e1 / R1 - c1, e2 / R2 - c2 + + def _component_responses(self, ver, catalog=None): + """Resolve responses identically for mean-c production and subtraction. + + Called within the catalogue's temporarily_read_data context. """ - e1, e2 = self._read_shear_cols(ver, "e1_col", "e2_col") if ver == "DES": - R1 = np.average(self.results[ver].dat_shear[self.cc[ver]["shear"]["R11"]]) - R2 = np.average(self.results[ver].dat_shear[self.cc[ver]["shear"]["R22"]]) + if catalog is None: + catalog = self.results[ver].dat_shear + R1 = np.average(catalog[self.cc[ver]["shear"]["R11"]]) + R2 = np.average(catalog[self.cc[ver]["shear"]["R22"]]) else: R1 = R2 = self.cc[ver]["shear"]["R"] - return (e1 - self.c1[ver]) / R1, (e2 - self.c2[ver]) / R2 + return R1, R2 @property def colors(self): diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 24323d15f..0fc4f3784 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -97,7 +97,32 @@ def _spectra(part, bins=(0, 0)): return {"ELL": ell, "EE": ee, "EB": eb, "BE": eb.copy(), "BB": bb} +class _ShearCatalogue: + """Override shear columns without modifying the source catalogue.""" + + def __init__(self, catalog, columns): + self.catalog = catalog + self.columns = columns + + def __getitem__(self, key): + if isinstance(key, str): + if key in self.columns: + return self.columns[key] + return self.catalog[key] + return _ShearCatalogue( + self.catalog[key], + {name: value[key] for name, value in self.columns.items()}, + ) + + class PseudoClMixin: + def _pseudo_cl_catalogue(self, ver): + """Read the catalogue with the same calibrated shear used for ξ±.""" + catalog = open_entry(self.cc[ver]["shear"]) + g1, g2 = self._calibrated_g(ver, catalog) + shear = self.cc[ver]["shear"] + return _ShearCatalogue(catalog, {shear["e1_col"]: g1, shear["e2_col"]: g2}) + # ---------------- Pseudo-Cl properties ---------------- # @property def pseudo_cls(self): @@ -226,7 +251,7 @@ def _measure_pseudo_cl_map(self, ver, nside, tomo_bin_a, tomo_bin_b): ) # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + cat_gal = self._pseudo_cl_catalogue(ver) # Get the tomographic bin cat_gal_a = self._get_tomographic_bin(params, cat_gal, tomo_bin_a) @@ -315,7 +340,7 @@ def _measure_pseudo_cl_catalog(self, ver, tomo_bin_a, tomo_bin_b): params = get_params_rho_tau(self.cc[ver]) # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + cat_gal = self._pseudo_cl_catalogue(ver) ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog( catalog=cat_gal, params=params, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b @@ -434,7 +459,7 @@ def calculate_pseudo_cl_inka_cov( self.print_cyan(f"Method used: {self.noise_bias_method}") params = get_params_rho_tau(self.cc[ver]) - cat_gal = open_entry(self.cc[ver]["shear"]) + cat_gal = self._pseudo_cl_catalogue(ver) lmin, lmax, b_lmax = spv_pseudo_cl.pseudo_cl_geometry(self.nside) b = self.get_namaster_bin(lmin, lmax, b_lmax) diff --git a/src/sp_validation/des_jackknife.py b/src/sp_validation/des_jackknife.py new file mode 100644 index 000000000..6dca990e0 --- /dev/null +++ b/src/sp_validation/des_jackknife.py @@ -0,0 +1,189 @@ +"""Exact refit-DES deletions from fixed-cell sufficient statistics. + +Storage is O(groups * cells), independent of parent catalogue length. Feed +selected branch chunks with add_branch; no parent-sized scratch arrays are kept. +""" + +import numpy as np + +from sp_validation import response + + +def pooled_weight(moments, shape, min_count): + """Reapply the concentric-square rule to retained count, sum R and sum e². + + Integral images make each window query O(1). All sparse cells are evaluated + together at each radius; with the fixed 20x20 production grid this is O(cells). + Original counts, not pooled counts, determine which cells need pooling. + """ + count, rsum, esum = moments + weight = np.divide( + rsum**2, count * esum, out=np.zeros_like(rsum), where=count * esum > 0 + ) + if min_count < 0: + raise ValueError("min_count must be nonnegative") + pending = np.flatnonzero(count < min_count) + if not len(pending): + return weight + # Sparse windows subtract large prefix sums. Extended precision avoids + # cancellation at the high-index edges (visible even on the small grids). + image = np.pad( + moments.astype(np.longdouble).reshape(3, *shape), + ((0, 0), (1, 0), (1, 0)), + ) + image = image.cumsum(axis=1).cumsum(axis=2) + for radius in range(1, max(shape)): + i, j = np.unravel_index(pending, shape) + lo_i, hi_i = np.maximum(i - radius, 0), np.minimum(i + radius + 1, shape[0]) + lo_j, hi_j = np.maximum(j - radius, 0), np.minimum(j + radius + 1, shape[1]) + sums = ( + image[:, hi_i, hi_j] + - image[:, lo_i, hi_j] + - image[:, hi_i, lo_j] + + image[:, lo_i, lo_j] + ) + ready = (sums[0] >= min_count) | (radius == max(shape) - 1) + n, r, e2 = sums[:, ready] + weight[pending[ready]] = np.divide( + r**2, n * e2, out=np.zeros_like(r), where=n * e2 > 0 + ) + pending = pending[~ready] + if not len(pending): + break + return weight + + +class DESJackknife: + """Accumulate raw moments, then apply each deletion's refitted cell weights. + + The caller supplies the full-sample edges, never deletion-dependent edges. + Per branch/cell we need count, sum of no-shear shapes (selection response), + and sum of that branch's shapes (joint response). The ns branch also stores + sum R_g and the three valid-training moments used by fit_w. + """ + + def __init__(self, labels, edges_s, edges_r, step, min_count=3): + self.labels = np.asarray(labels) + if len(labels) < 2: + raise ValueError("Response jackknife requires at least two spatial groups") + self.edges = edges_s, edges_r + self.shape = len(edges_s) - 1, len(edges_r) - 1 + self.cells = np.prod(self.shape) + self.step, self.min_count = step, min_count + shape = len(labels), self.cells + self.sums = {b: np.zeros((5, *shape)) for b in response.BRANCHES} + self.rg = np.zeros((4, *shape)) + self.training = np.zeros((3, *shape)) + + def add_branch(self, b, groups, snr, size, ns_e, e, rg=None): + """Add selected rows of one branch; rg aligns to selected ns rows only.""" + finite = np.isfinite(snr) & np.isfinite(size) + cell = response.cell_index(snr, self.edges[0]) * self.shape[ + 1 + ] + response.cell_index(size, self.edges[1]) + index = np.searchsorted(self.labels, groups) * self.cells + cell + + def add(target, values, valid=finite): + ix = index[valid] + if not len(ix): + return + v = None if values is None else values[valid] + lo, hi = ix.min(), ix.max() + 1 + flat = target.reshape(-1) + if hi - lo <= 2 * len(ix): + # Sorted spatial catalogues usually touch only a few groups + # per chunk. Never clear a groups*cells scratch per chunk: + # cap the dense span at O(chunk rows), otherwise scatter. + flat[lo:hi] += np.bincount(ix - lo, weights=v, minlength=hi - lo) + else: + np.add.at(flat, ix, 1 if v is None else v) + + add(self.sums[b][0], None) + for i in range(2): + add(self.sums[b][1 + i], ns_e[i]) + add(self.sums[b][3 + i], e[i]) + if b == "ns": + for i, values in enumerate(rg.reshape(4, -1)): + add(self.rg[i], values) + r = 0.5 * (rg[0, 0] + rg[1, 1]) + valid = finite & np.isfinite(e).all(axis=0) & np.isfinite(r) + add(self.training[0], None, valid) + add(self.training[1], r, valid) + add(self.training[2], np.mean(e**2, axis=0), valid) + + def deleted_weight(self, group): + """Return the retained table, useful for reference comparisons.""" + moments = self.training.sum(axis=1) - self.training[:, group] + return pooled_weight(moments, self.shape, self.min_count).reshape(self.shape) + + def jackknife(self): + """Return the same response/c estimates and covariance as the row path.""" + totals = {b: x.sum(axis=1) for b, x in self.sums.items()} + training_total = self.training.sum(axis=1) + rg_total = self.rg.sum(axis=1) + samples = [] + for k in range(len(self.labels)): + w = pooled_weight( + training_total - self.training[:, k], self.shape, self.min_count + ) + means = {} + den = {} + for b in response.BRANCHES: + weighted = (totals[b] - self.sums[b][:, k]) @ w + den[b] = weighted[0] + if den[b] <= 0: + raise ValueError( + f"Empty weighted population after spatial deletion: {b}" + ) + means[b] = weighted[1:] / den[b] + shear = ((rg_total - self.rg[:, k]) @ w / den["ns"]).reshape(2, 2) + selection, joint = np.empty((2, 2)), np.empty((2, 2)) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + selection[:, j] = (means[p][:2] - means[m][:2]) / (2 * self.step) + joint[:, j] = (means[p][2:] - means[m][2:]) / (2 * self.step) + total = shear + selection + ce = means["ns"][:2] + samples.append( + dict( + shear=shear, + selection=selection, + total=total, + joint=joint, + c_e=ce, + c_g=np.linalg.solve(total, ce), + ) + ) + estimates = {key: np.array([s[key] for s in samples]) for key in samples[0]} + vectors = np.concatenate( + [x.reshape(len(self.labels), -1) for x in estimates.values()], axis=1 + ) + delta = vectors - vectors.mean(axis=0) + cov = (len(self.labels) - 1) / len(self.labels) * delta.T @ delta + return self.labels, estimates, cov + + +def accumulate(gm, groups, chunk_size=500_000): + """Read an existing metacal parent in bounded chunks, without copying it.""" + groups = np.asarray(groups) + if len(groups) != len(gm.ns["g1"]): + raise ValueError("Spatial groups must align to the common parent rows") + # np.unique(return_inverse=True) on 350M rows would itself require GBs. + labels = np.empty(0, dtype=groups.dtype) + for start in range(0, len(groups), chunk_size): + labels = np.union1d(labels, np.unique(groups[start : start + chunk_size])) + acc = DESJackknife(labels, gm.W[0], gm.W[1], gm._step, gm._des_min_count) + for b in response.BRANCHES: + selected = gm.mask_dict[b] + d = getattr(gm, b) + for start in range(0, len(selected), chunk_size): + idx = selected[start : start + chunk_size] + acc.add_branch( + b, + groups[idx], + d["flux"][idx] / d["flux_err"][idx], + d["T"][idx] / d["Tpsf"][idx], + np.array([x[idx] for x in gm.branches["ns"]]), + np.array([x[idx] for x in gm.branches[b]]), + gm.R_shear[:, :, start : start + chunk_size] if b == "ns" else None, + ) + return acc diff --git a/src/sp_validation/image_sim_config.py b/src/sp_validation/image_sim_config.py new file mode 100644 index 000000000..a0a368cc6 --- /dev/null +++ b/src/sp_validation/image_sim_config.py @@ -0,0 +1,33 @@ +"""Image-simulation calibration policy (no catalogue I/O).""" + +from copy import deepcopy + + +def calibration_config(config, w_cols, source="external", product=None): + """Use a named data calibration table by default for DES image sims. + + Explicit ``source='self'`` retains sample-fitted W as a diagnostic. + Non-DES runs retain their existing calibration configuration. + """ + config = deepcopy(config) + if "w_des" not in w_cols: + return config + if source not in ("external", "self"): + raise ValueError("des_weight_source must be external or self") + if source == "external" and ( + not isinstance(product, dict) + or not product.get("name") + or not product.get("path") + ): + raise ValueError( + "DES image sims require des_weight_product with name and path; " + "des_weight_source: self is diagnostic only" + ) + cm = config["metacal"] + cm.pop("global_R_weight", None) + cm.update(weight_scheme="des", w_col="w_des", additive_correction=False) + if source == "external": + cm["des_weight_product"] = product.copy() + else: + cm.pop("des_weight_product", None) + return config diff --git a/src/sp_validation/image_sims.py b/src/sp_validation/image_sims.py index e259109d2..ae879d1d1 100644 --- a/src/sp_validation/image_sims.py +++ b/src/sp_validation/image_sims.py @@ -51,8 +51,14 @@ def _load_cat(path, w_cols): scheme in a multi-weight run. """ with fits.open(path) as hdul: + if hdul[0].header.get("RAPPLIED", False): + from sp_validation.response import validate_weight_scheme + + for col in w_cols: + validate_weight_scheme(hdul[0].header["WGHTSCH"], col) data = hdul[1].data cat = { + "weight_scheme": hdul[0].header.get("WGHTSCH"), "ra": data["RA"].copy(), "dec": data["Dec"].copy(), "e1": data["e1"].copy(), @@ -65,6 +71,27 @@ def _load_cat(path, w_cols): if _is_unweighted(scheme) else data[scheme].copy() ) + if cat["weight_scheme"] is not None: + if hdul[0].header.get("MEANSUB", False): + raise ValueError( + "Constant-shear simulations must not subtract branch means" + ) + cat["raw"] = np.array([data["e1_uncal"], data["e2_uncal"]]) + cat["tiles"] = np.char.strip(np.asarray(data["TILE_ID"]).astype(str)) + if "R_JK" not in hdul: + raise ValueError( + "Modern m uncertainties require shared-tile R_JK responses" + ) + jk = hdul["R_JK"].data + cat["jk_labels"] = np.char.strip(np.asarray(jk["TILE_ID"]).astype(str)) + cat["jk_R"] = np.array( + [[jk[f"R_{i}{j}"] for j in (1, 2)] for i in (1, 2)] + ).transpose(2, 0, 1) + cat["W_policy"] = hdul[0].header.get("WPOLICY", "unknown") + if cat["W_policy"] == "refit": + # These means include each deletion's refitted numerator weights, + # not merely a refitted response applied to fixed-W means. + cat["jk_g"] = np.array([jk["c_g1"], jk["c_g2"]]).T return cat @@ -97,13 +124,14 @@ class ImageSimMBias: (required); ``"none"`` (or a ``null`` entry) means unit weights, any other entry is a FITS column name. The **first** entry is the primary result surfaced at the top level of ``run()``'s output. Our - fiducial run leads with the unweighted scheme (``["none", ...]``), - per the #227 verdict that shape weights are excluded from sim - calibration; the unweighted m also avoids the ``cov(w, e)`` residual - weighted estimators carry on constant-shear sims. + modern production run uses the weight matching the stored response + (DES case d: ``["w_des"]``) and ``pair_match=False``. + Symmetric pair weights with a full-population response are refused. - w_col : str or None, *deprecated* single weight scheme; accepted for back-compat and used as ``[w_col]`` only when ``w_cols`` is absent. - - n_bootstrap : int, number of bootstrap resamples for errors (required) + - conditional_bootstrap : bool, optional fixed-response object-bootstrap + diagnostic for modern products (default False); never the headline error + - n_bootstrap : int, number of conditional diagnostic draws (required) - bootstrap_seed : int, seed for the per-pair bootstrap RNG (required); makes the bootstrap errors bit-reproducible. The resample indices are drawn once per pair and shared across every weight scheme, so the @@ -152,14 +180,24 @@ def load_catalogs(self, verbose=True): grids_dir = self.cfg["grids_dir"] num = self.cfg["num"] cat_name = self.cfg.get("catalog_name", "shape_catalog_cut_ngmix.fits") + # Grid sims live in ``{branch}_grid_{num}``; other families (e.g. + # blended) in ``{branch}_{num}``. + sims_type = self.cfg.get("sims_type", "grid") + suffix = f"_grid_{num}" if sims_type == "grid" else f"_{num}" # ``sim_names`` (incl. the unsheared reference) comes from the config's # branch map. The +g/-g pool estimator pairs the sheared sims directly; # the reference is loaded for completeness and null-test diagnostics. for name in self.sim_names: - path = f"{grids_dir}/{name}_grid_{num}/{cat_name}" + path = f"{grids_dir}/{name}{suffix}/{cat_name}" if verbose: print(f" Loading {path}") self.cats[name] = _load_cat(path, self.w_cols) + if self.pair_match and self.cats[name]["weight_scheme"] is not None: + raise ValueError( + "Modern response products require pair_match=False: headline m " + "uses full populations and separately normalized branch weights; " + "symmetric pair weights do not match the stored response." + ) if verbose: print(f" {len(self.cats[name]['ra'])} objects") @@ -228,24 +266,13 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): e_p = self.cats[name_p][e_key][idx_p] e_m = self.cats[name_m][e_key][idx_m] - # Draw the bootstrap resample indices *once*, before the weight-scheme - # loop, and reuse them for every scheme -- the schemes then differ only - # in their weighting, never in their draws (so a scheme comparison is a - # clean weighting comparison). Pre-drawing the full ``(n_boot, n)`` block - # in one call is bit-identical to drawing ``rng.integers(0, n, n)`` once - # per replicate (numpy fills the block row-major), so the numbers match a - # single-scheme, per-iteration bootstrap to the last bit. - rng = np.random.default_rng(seed=self.boot_seed) - if self.pair_match: - n = len(e_p) - ib = rng.integers(0, n, (self.n_boot, n)) - else: - n_p, n_m = len(e_p), len(e_m) - ib_p = rng.integers(0, n_p, (self.n_boot, n_p)) - ib_m = rng.integers(0, n_m, (self.n_boot, n_m)) + # Generate one draw at a time: no catalogue-sized n_boot x N array. + # Reset the seed per scheme so comparisons use identical draws. + n_p, n_m = len(e_p), len(e_m) res = {} for scheme in self.w_cols: + rng = np.random.default_rng(seed=self.boot_seed) w_p = self.cats[name_p]["w"][scheme][idx_p] w_m = self.cats[name_m]["w"][scheme][idx_m] m_boot = np.empty(self.n_boot) @@ -265,8 +292,9 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): # sims, so the per-object cancellation in `d` is preserved in # the error estimate. for i in range(self.n_boot): - m_boot[i] = np.average(d[ib[i]], weights=w[ib[i]]) - c_boot[i] = np.average(s[ib[i]], weights=w[ib[i]]) + ib = rng.integers(0, n_p, n_p) + m_boot[i] = np.average(d[ib], weights=w[ib]) + c_boot[i] = np.average(s[ib], weights=w[ib]) else: # No matching: difference/sum the two independent weighted means. mean_ep = np.average(e_p, weights=w_p) @@ -278,8 +306,10 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): # Unpaired bootstrap: the +g and -g arrays generally differ in # length, so each side is resampled independently per replicate. for i in range(self.n_boot): - ep_b = np.average(e_p[ib_p[i]], weights=w_p[ib_p[i]]) - em_b = np.average(e_m[ib_m[i]], weights=w_m[ib_m[i]]) + ib_p = rng.integers(0, n_p, n_p) + ib_m = rng.integers(0, n_m, n_m) + ep_b = np.average(e_p[ib_p], weights=w_p[ib_p]) + em_b = np.average(e_m[ib_m], weights=w_m[ib_m]) m_boot[i] = (ep_b - em_b) / (2 * self.g_in) - 1 c_boot[i] = (ep_b + em_b) / 2 @@ -287,9 +317,109 @@ def _m_c_pair(self, name_p, name_m, comp, verbose=True): return res + def _run_tile_jackknife(self, verbose): + """Delete the same parent tile on every external-shear branch. + + R_JK carries responses recomputed upstream from all five metacal + selections, including sheared-only selected rows. Raw selected shapes + supply each deletion's numerator; never invert the full-sample response + to approximate a deleted response. + """ + cats = [self.cats[name] for p, m, _ in self.pairs for name in (p, m)] + labels = np.sort(cats[0]["jk_labels"]) + if len(labels) < 2: + raise ValueError("At least two shared tiles are required") + for cat in cats: + if not np.array_equal(np.sort(cat["jk_labels"]), labels): + raise ValueError( + "External branches must have identical parent R_JK tiles" + ) + if not np.all(np.isin(cat["tiles"], labels)): + raise ValueError("Selected tiles are missing from parent R_JK") + order = [ + key for _, _, comp in self.pairs for key in (f"m{comp + 1}", f"c{comp + 1}") + ] + results = dict( + weights={}, + estimator="full_population_branch_normalized", + uncertainty="shared_tile_response_jackknife", + W_policy=cats[0]["W_policy"], + covariance_order=order, + jackknife_tiles=labels.tolist(), + ) + for scheme in self.w_cols: + estimates, deleted = [], [] + for p, m, comp in self.pairs: + means, jk_means = [], [] + for name in (p, m): + cat = self.cats[name] + w = cat["w"][scheme] + means.append(np.average(cat[f"e{comp + 1}"], weights=w)) + if cat["W_policy"] == "refit": + by_label = dict(zip(cat["jk_labels"], cat["jk_g"], strict=True)) + jk_means.append(np.array([by_label[t][comp] for t in labels])) + continue + by_label = dict(zip(cat["jk_labels"], cat["jk_R"], strict=True)) + calibrated = [] + for tile in labels: + keep = cat["tiles"] != tile + raw_mean = np.average( + cat["raw"][:, keep], axis=1, weights=w[keep] + ) + calibrated.append( + np.linalg.solve(by_label[tile], raw_mean)[comp] + ) + jk_means.append(np.asarray(calibrated)) + estimates.extend( + ( + (means[0] - means[1]) / (2 * self.g_in) - 1, + (means[0] + means[1]) / 2, + ) + ) + deleted.extend( + ( + (jk_means[0] - jk_means[1]) / (2 * self.g_in) - 1, + (jk_means[0] + jk_means[1]) / 2, + ) + ) + draws = np.array(deleted).T + delta = draws - draws.mean(axis=0) + cov = (len(labels) - 1) / len(labels) * delta.T @ delta + result = dict(zip(order, map(float, estimates), strict=True)) + result.update( + { + f"{key}_err": float(error) + for key, error in zip(order, np.sqrt(np.diag(cov)), strict=True) + } + ) + result["mc_covariance"] = cov.tolist() + result["mc_jackknife"] = draws.tolist() + results["weights"][scheme] = result + if verbose: + print( + f"[{scheme}] shared-tile response jackknife ({len(labels)} tiles)" + ) + for key in order: + print(f" {key} = {result[key]:+.6f} ± {result[key + '_err']:.6f}") + results.update(results["weights"][self.w_cols[0]]) + # Optional, explicitly conditional diagnostic; never replace science errors. + if self.cfg.get("conditional_bootstrap", False): + diagnostic = {scheme: {} for scheme in self.w_cols} + for p, m, comp in self.pairs: + for scheme, values in self._m_c_pair(p, m, comp, False).items(): + diagnostic[scheme][f"m{comp + 1}_err"] = float(values[1]) + diagnostic[scheme][f"c{comp + 1}_err"] = float(values[3]) + results["conditional_object_bootstrap_fixed_response"] = diagnostic + return results + def run(self, verbose=True): """Compute m and c for both shear components and every weight scheme. + Modern products use shared-tile, response-recalibrating jackknife errors + and export mc_covariance in covariance_order, including m/c cross terms. + W_policy records whether the upstream deleted responses held W fixed. + Legacy object-bootstrap errors are labelled conditional on fixed R. + Returns ------- dict @@ -299,7 +429,20 @@ def run(self, verbose=True): reader that wants the headline m/c never has to know the scheme name. """ - results = {"weights": {scheme: {} for scheme in self.w_cols}} + modern = [cat.get("weight_scheme") is not None for cat in self.cats.values()] + if any(modern): + if not all(modern) or self.pair_match: + raise ValueError( + "Modern response products require full populations on all branches" + ) + return self._run_tile_jackknife(verbose) + results = { + "weights": {scheme: {} for scheme in self.w_cols}, + "estimator": "legacy_symmetric_pairs" + if self.pair_match + else "full_population", + "uncertainty": "conditional_object_bootstrap_fixed_response", + } for name_p, name_m, comp in self.pairs: label = f"g{comp + 1}" if verbose: diff --git a/src/sp_validation/io.py b/src/sp_validation/io.py index b95b90dfa..ac12238a5 100644 --- a/src/sp_validation/io.py +++ b/src/sp_validation/io.py @@ -365,12 +365,29 @@ def open_catalogue(path, hdu=None, column_map=None): return catalogue.table() -def open_entry(entry): - """Return the catalogue a config block names (e.g. a cat_config ``shear``). +def validate_calibrated_entry(entry): + """Validate response/weight identity, also for file-based reader adapters.""" + if "w_col" in entry and not h5py.is_hdf5(entry["path"]): + with fits.open(entry["path"], memmap=True) as hdus: + header = hdus[0].header + if header.get("RAPPLIED", False): + from sp_validation.response import validate_weight_scheme + + validate_weight_scheme(header["WGHTSCH"], entry["w_col"]) + if ( + entry.get("R", 1) != 1 + or entry.get("R11") is not None + or entry.get("R22") is not None + ): + raise ValueError( + "Response already applied: cat_config R must be 1 " + "with no per-object response columns" + ) - The block's ``path``, optional ``hdu`` and optional ``column_map`` go to - ``open_catalogue``. - """ + +def open_entry(entry): + """Read a config entry, checking calibrated-response/weight metadata.""" + validate_calibrated_entry(entry) return open_catalogue( entry["path"], hdu=entry.get("hdu"), column_map=entry.get("column_map") ) diff --git a/src/sp_validation/response.py b/src/sp_validation/response.py new file mode 100644 index 000000000..58825d513 --- /dev/null +++ b/src/sp_validation/response.py @@ -0,0 +1,188 @@ +"""Weighted metacal response primitives. + +Definitions: Martin Kilbinger's 6 October 2026 slide, p14 (S1), and +scripts/validation/wdes_response/wdes_response.py, case d. +Spatial deletion follows his response_jackknife, with fixed-W +comparisons and retained-sample refits in calibration.metacal. +No detection response is inferred from aligned metacal branches. +""" + +import numpy as np + +BRANCHES = ("ns", "p1", "m1", "p2", "m2") + + +def branch_error_columns(data): + """Carry available sheared G_ERRs through extraction without requiring them. + + Historical merged products omitted them. DES can still use those products; + modern IV independently requires a complete set and never freezes weights. + """ + return { + key: data[key] + for shear in ("1P", "1M", "2P", "2M") + for i in (1, 2) + if (key := f"NGMIX_G{i}_ERR_{shear}") in data.dtype.names + } + + +def cell_index(x, edges): + """Right-closed cells; out-of-range finite values use nearest edge cell.""" + return np.clip(np.searchsorted(edges, x, side="left") - 1, 0, len(edges) - 2) + + +def evaluate_w(table, snr, size): + """Evaluate one fixed table, including rows absent from the training sample. + + Unpooled empty/invalid cells and nonfinite coordinates get zero weight. + """ + es, er, values, _ = table + w = values[cell_index(snr, es), cell_index(size, er)].copy() + w[~np.isfinite(snr) | ~np.isfinite(size) | ~np.isfinite(w)] = 0 + return w + + +def fit_w(snr, size, e, response, edges_s, edges_r, min_count=0): + """Fit W = mean(trace R/2)^2 / mean(|e|^2/2), unweighted per cell. + + Uses second moments, not centered variances. Fitting and evaluation share + the same boundary convention. Below min_count, pool moments over the + smallest clipped concentric square of cells reaching min_count; if the + entire training set is smaller, use its moments. Populated cells are exact. + The returned counts are unmerged training counts. + """ + shape = (len(edges_s) - 1, len(edges_r) - 1) + idx = cell_index(snr, edges_s) * shape[1] + cell_index(size, edges_r) + valid = np.isfinite(snr) & np.isfinite(size) + valid &= np.all(np.isfinite(e), axis=0) & np.isfinite(response) + idx = idx[valid] + count = np.bincount(idx, minlength=np.prod(shape)) + rsum = np.bincount(idx, weights=response[valid], minlength=len(count)) + esum = np.bincount( + idx, weights=np.mean(e[:, valid] ** 2, axis=0), minlength=len(count) + ) + w = np.divide( + rsum**2, count * esum, out=np.zeros_like(rsum), where=(count * esum) > 0 + ) + if min_count < 0: + raise ValueError("min_count must be nonnegative") + # Smallest concentric square in cell-index space (clipped at the edges). + # Pool sufficient statistics, never weights; retain the original counts. + if min_count: + counts, rsums, esums = (x.reshape(shape) for x in (count, rsum, esum)) + for i, j in np.argwhere(counts < min_count): + for radius in range(1, max(shape)): + window = ( + slice(max(0, i - radius), min(shape[0], i + radius + 1)), + slice(max(0, j - radius), min(shape[1], j + radius + 1)), + ) + n = counts[window].sum() + if n >= min_count or radius == max(shape) - 1: + r, e2 = rsums[window].sum(), esums[window].sum() + w[i * shape[1] + j] = r**2 / (n * e2) if n * e2 > 0 else 0 + break + return edges_s, edges_r, w.reshape(shape), count.reshape(shape) + + +def responses(branches, masks, weights, rg, step, keep=None): + """S1 split and joint derivatives, each mean separately normalized. + + rg is the per-object response on no-shear-selected rows, aligned to + masks['ns']. All other arrays are aligned to the common parent rows. + """ + selected = {b: idx if keep is None else idx[keep[idx]] for b, idx in masks.items()} + + def mean(data, b): + idx = selected[b] + w = weights[b][idx] + if not len(idx) or not np.isfinite(w).all() or w.sum() <= 0: + raise ValueError(f"Empty or invalid weighted population: {b}") + return np.array([np.average(x[idx], weights=w) for x in data]) + + ns = masks["ns"] + k = np.ones(len(ns), bool) if keep is None else keep[ns] + shear = np.average(rg[:, :, k], axis=2, weights=weights["ns"][ns][k]) + selection = np.empty((2, 2)) + joint = np.empty((2, 2)) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + selection[:, j] = (mean(branches["ns"], p) - mean(branches["ns"], m)) / ( + 2 * step + ) + joint[:, j] = (mean(branches[p], p) - mean(branches[m], m)) / (2 * step) + total = shear + selection + ce = mean(branches["ns"], "ns") + cg = np.linalg.solve(total, ce) + return dict( + shear=shear, selection=selection, total=total, joint=joint, c_e=ce, c_g=cg + ) + + +def jackknife(branches, masks, weights, rg, step, groups): + """Delete common spatial groups in all branches; conditional on fixed W. + + Covariance order: shear, selection, total, joint (row-major), c_e, c_g. + Extends Martin's response_jackknife to additive bias and full covariance. + """ + if len(groups) != len(branches["ns"][0]): + raise ValueError("Spatial groups must align to the common parent rows") + labels, group_index = np.unique(groups, return_inverse=True) + if len(labels) < 2: + raise ValueError("Response jackknife requires at least two spatial groups") + + # Fixed-W sufficient sums: O(N) rather than rereading N rows K times. + def deleted_sum(values, index): + sums = np.bincount(index, weights=values, minlength=len(labels)) + return sums.sum() - sums + + den = {} + for b in masks: + idx = masks[b] + den[b] = deleted_sum(weights[b][idx], group_index[idx]) + if np.any(den[b] <= 0): + raise ValueError(f"Empty weighted population after spatial deletion: {b}") + + def means(data, b): + idx = masks[b] + return np.array( + [ + deleted_sum(x[idx] * weights[b][idx], group_index[idx]) / den[b] + for x in data + ] + ).T + + ns = masks["ns"] + shear = np.empty((len(labels), 2, 2)) + selection = np.empty_like(shear) + joint = np.empty_like(shear) + for i in range(2): + for j in range(2): + shear[:, i, j] = ( + deleted_sum(rg[i, j] * weights["ns"][ns], group_index[ns]) / den["ns"] + ) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + selection[:, :, j] = (means(branches["ns"], p) - means(branches["ns"], m)) / ( + 2 * step + ) + joint[:, :, j] = (means(branches[p], p) - means(branches[m], m)) / (2 * step) + total = shear + selection + ce = means(branches["ns"], "ns") + cg = np.linalg.solve(total, ce[..., None])[..., 0] + estimates = dict( + shear=shear, selection=selection, total=total, joint=joint, c_e=ce, c_g=cg + ) + vectors = np.concatenate( + [estimates[k].reshape(len(labels), -1) for k in estimates], axis=1 + ) + delta = vectors - vectors.mean(axis=0) + cov = (len(labels) - 1) / len(labels) * delta.T @ delta + return labels, estimates, cov + + +def validate_weight_scheme(scheme, w_col): + """Reject statistic/calibration mismatches, rather than relabel a response.""" + if scheme not in ("des", "iv"): + raise ValueError("weight_scheme must be des or iv") + if w_col != f"w_{scheme}": + raise ValueError( + f"Calibration weight_scheme={scheme} disagrees with w_col={w_col}" + ) diff --git a/src/sp_validation/response_diagnostics.py b/src/sp_validation/response_diagnostics.py new file mode 100644 index 000000000..9ed93d317 --- /dev/null +++ b/src/sp_validation/response_diagnostics.py @@ -0,0 +1,189 @@ +"""Response-only diagnostics, not cosmological signal measurements. + +Pair products use raw per-object R_g, not total selection/weight response. +They cannot certify adequacy of a global total response on their own (#23). +""" + +import numpy as np + + +def pair_products(ra, dec, weight, rg, min_sep=1, max_sep=120, nbins=8): + """Symmetrized angle-binned _ww via TreeCorr. + + Coordinates are degrees, separations arcminutes. Nonzero min_sep excludes + self-pairs; cross-scalar correlations include both pair orientations. + Returns all 16 tensor products, not just the scalar trace approximation. + """ + import treecorr + + scalar = rg.reshape(4, -1) + cats = [ + treecorr.Catalog(ra=ra, dec=dec, ra_units="deg", dec_units="deg", w=weight, k=x) + for x in scalar + ] + products = np.empty((nbins, 4, 4)) + for i in range(4): + for j in range(i, 4): + corr = treecorr.KKCorrelation( + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + sep_units="arcmin", + bin_slop=0, + ) + corr.process(cats[i], None if i == j else cats[j], num_threads=4) + products[:, i, j] = products[:, j, i] = np.where( + corr.weight > 0, corr.xi, np.nan + ) + means = np.average(scalar, axis=1, weights=weight) + return dict( + theta=np.exp(corr.meanlogr), + products=products, + global_products=np.outer(means, means), + pair_weight=corr.weight, + ) + + +def scalar_pair_response( + ra, + dec, + weight, + r, + *, + patch=None, + patch_centers=None, + min_sep=1, + max_sep=250, + nbins=20, + bin_slop=0.1, +): + """Stage-1 _ww / _w**2 - 1, r = trace(R_g)/2. + + Delete the identical patch from pair and global statistics, then form each + ratio. This includes covariance of the pair and global response, unlike + an error bar on the pair product alone. Science selection and weights are + fixed inputs; selection/weight derivatives and anisotropic mixing aren't + measured here. No ellipticity correlation is computed. + """ + import treecorr + + cat = treecorr.Catalog( + ra=ra, + dec=dec, + ra_units="deg", + dec_units="deg", + w=weight, + k=r, + patch=patch, + patch_centers=patch_centers, + ) + corr = treecorr.KKCorrelation( + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + sep_units="arcmin", + bin_slop=bin_slop, + var_method="jackknife", + cross_patch_weight="simple", + ) + corr.process(cat, num_threads=4) + pairs, _ = treecorr.build_multi_cov_design_matrix( + [corr], "jackknife", cross_patch_weight="simple" + ) + group = cat.patch + den = np.bincount(group, weights=weight, minlength=cat.npatch) + num = np.bincount(group, weights=weight * r, minlength=cat.npatch) + global_deleted = (num.sum() - num) / (den.sum() - den) + global_r = num.sum() / den.sum() + deleted = pairs / global_deleted[:, None] ** 2 - 1 + delta = deleted - deleted.mean(axis=0) + covariance = (len(deleted) - 1) / len(deleted) * delta.T @ delta + return dict( + theta=np.exp(corr.meanlogr), + pair=corr.xi, + global_r=global_r, + fraction=corr.xi / global_r**2 - 1, + deleted=deleted, + global_deleted=global_deleted, + covariance=covariance, + error=np.sqrt(np.diag(covariance)), + pair_weight=corr.weight, + ) + + +def plot_response(gm, parent_groups, ra, dec, output): + """Weighted/unweighted R_g, spatial split/joint samples, and patch map.""" + import matplotlib.pyplot as plt + + selected = gm.mask_dict["ns"] + w = gm.branch_weights["ns"][selected] + labels, jk, _ = gm.response_jackknife(parent_groups) + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + ax = axes[i, j] + x = gm.R_shear[i, j] + limits = np.percentile(x, [0.5, 99.5]) + ax.hist( + x, + bins=60, + range=limits, + density=True, + histtype="step", + label="Unweighted", + ) + ax.hist( + x, + bins=60, + range=limits, + weights=w, + density=True, + histtype="step", + label="DES weighted", + ) + ax.set(xlabel=f"Per-object Rg,{i + 1}{j + 1}", ylabel="Density") + ax.legend() + fig.tight_layout() + fig.savefig(output / "response_distributions.png", dpi=160) + plt.close(fig) + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + ax = axes[i, j] + for key in ("selection", "total", "joint"): + ax.plot(np.arange(len(labels)), jk[key][:, i, j], ".", label=key) + ax.set(xlabel="Deleted spatial group", ylabel=f"Response {i + 1}{j + 1}") + ax.legend() + fig.tight_layout() + fig.savefig(output / "response_jackknife.png", dpi=160) + plt.close(fig) + groups = parent_groups[selected] + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + values, x, y = [], [], [] + for label in labels: + keep = groups == label + if not keep.any(): + continue + x.append(np.mean(ra[keep])) + y.append(np.mean(dec[keep])) + values.append(np.average(gm.R_shear[i, j, keep], weights=w[keep])) + im = axes[i, j].scatter(x, y, c=values, s=45) + fig.colorbar(im, ax=axes[i, j], label=f"Mean Rg,{i + 1}{j + 1}") + axes[i, j].set(xlabel="RA [deg]", ylabel="Dec [deg]") + fig.tight_layout() + fig.savefig(output / "response_map.png", dpi=160) + plt.close(fig) + fig, axes = plt.subplots(2, 2, figsize=(10, 7)) + for i in range(2): + for j in range(2): + delta = jk["total"][:, i, j] - jk["joint"][:, i, j] + axes[i, j].plot(np.arange(len(labels)), delta, ".") + axes[i, j].axhline(gm.R[i, j] - gm.R_joint[i, j], color="k", lw=0.8) + axes[i, j].set( + xlabel="Deleted spatial group", ylabel=f"Split − joint, {i + 1}{j + 1}" + ) + fig.tight_layout() + fig.savefig(output / "split_joint.png", dpi=160) + plt.close(fig) diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 2da5ccb3b..6490aeade 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -154,6 +154,7 @@ def _columns(self, block): def __call__(self, block): if block not in self._cache: entry = self._info[block] + io.validate_calibrated_entry(entry) with io.Catalogue( entry["path"], hdu=entry.get("hdu"), column_map=entry.get("column_map") ) as catalogue: diff --git a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py index c3c910abc..66877c429 100644 --- a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py +++ b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py @@ -136,6 +136,10 @@ def test_fresh_cv_merges_tomographic_covariance(tmp_path, monkeypatch): k: np.zeros(2) for k in ("ra_col", "dec_col", "e1_col", "e2_col", "w_col") }, ) + # Shear calibration is not under test: read the stub catalogue as is. + monkeypatch.setattr( + cv, "_pseudo_cl_catalogue", lambda ver: pseudo_cl.open_entry(None) + ) monkeypatch.setattr( pseudo_cl.spv_pseudo_cl, "get_field_and_workspace_from_catalog", diff --git a/src/sp_validation/tests/test_additive_response.py b/src/sp_validation/tests/test_additive_response.py new file mode 100644 index 000000000..0b58fa785 --- /dev/null +++ b/src/sp_validation/tests/test_additive_response.py @@ -0,0 +1,35 @@ +"""Additive-bias producer and consumer must agree on component units.""" + +from contextlib import nullcontext +from types import SimpleNamespace + +import numpy as np + +from sp_validation.cosmo_val.core import CosmologyValidation + + +def test_des_additive_bias_producer_to_consumer(): + obj = object.__new__(CosmologyValidation) + obj.versions = ["DES"] + obj.cc = { + "DES": { + "shear": dict( + e1_col="e1", e2_col="e2", w_col="w", R=1, R11="R11", R22="R22" + ) + } + } + data = dict( + e1=np.array([0.12, -0.04]), + e2=np.array([0.08, 0.16]), + w=np.ones(2), + R11=np.full(2, 0.8), + R22=np.full(2, 0.7), + ) + obj._results = { + "DES": SimpleNamespace(dat_shear=data, temporarily_read_data=nullcontext) + } + obj.print_start = obj.print_done = obj.print_magenta = lambda *args: None + obj.calculate_additive_bias() + np.testing.assert_allclose([obj.c1["DES"], obj.c2["DES"]], [0.04 / 0.8, 0.12 / 0.7]) + g = obj._calibrated_g("DES") + np.testing.assert_allclose(np.average(g, axis=1, weights=data["w"]), 0, atol=1e-16) diff --git a/src/sp_validation/tests/test_calibration.py b/src/sp_validation/tests/test_calibration.py index fbe787854..f6ab0af57 100644 --- a/src/sp_validation/tests/test_calibration.py +++ b/src/sp_validation/tests/test_calibration.py @@ -51,6 +51,7 @@ def make_fake_gal_metacal(R=R, g1=G1, g2=G2, w=W, mask=MASK): populated, so the test exercises the pure linear algebra and nothing else. """ return types.SimpleNamespace( + _legacy_response=True, # These literals characterize released legacy c/scatter. R=np.asarray(R, dtype=float), ns={ "g1": np.asarray(g1, dtype=float), diff --git a/src/sp_validation/tests/test_column_schema.py b/src/sp_validation/tests/test_column_schema.py index 6181c886e..16128e93c 100644 --- a/src/sp_validation/tests/test_column_schema.py +++ b/src/sp_validation/tests/test_column_schema.py @@ -39,7 +39,34 @@ "R", "R11", "R22", - # FITS header keywords + # FITS header keywords (response/c units, weighting and spatial products) + "CE_{}", + "CG_{}", + "CMUNITS", + "DESMERGE", + "DESNMIN", + "IVPOLICY", + "MCSTEP", + "MEANSUB", + "MIG_{}", + "N_{}", + "ORDER", + "RAPPLIED", + "RJ_{}{}", + "R_{}{}", # derived response-product matrix, not parent shape columns + "R_JK", # response-product FITS extension name + "RLEGACY", + "SW_{}", + "WGHTSCH", + "WPOLICY", + "WSNRCOL", + "WSIZDEF", + "WSOURCE", + "WPRODUCT", + "WPRODPTH", + "W_DES", # response-product FITS images, not parent catalogue columns + "W_COUNT", + "W_EDGES", "EXTNAME", "R_S11", "R_S12", diff --git a/src/sp_validation/tests/test_configured_response.py b/src/sp_validation/tests/test_configured_response.py new file mode 100644 index 000000000..650cb616d --- /dev/null +++ b/src/sp_validation/tests/test_configured_response.py @@ -0,0 +1,126 @@ +"""Global metacal response must use the weights of the shear statistic.""" + +from pathlib import Path + +import numpy as np +import yaml + +from sp_validation import calibration +from sp_validation.calibration import metacal + +GAMMA = np.array([0.02, -0.01]) +STEP = 0.01 +POPULATIONS = [(0.6, 1.0), (0.9, 3.0)] # (response, statistic weight) +N_PER_POP = 400 # Even: intrinsic ellipticities come in +/- pairs. + + +def _repository_root(): + root = Path(__file__).resolve().parents[3] + # A cross-check can import another checkout via PYTHONPATH while keeping + # this test file. Use that checkout's configuration as well as its code. + source_root = Path(calibration.__file__).resolve().parents[2] + if source_root != root and (source_root / "pyproject.toml").is_file(): + return source_root + return root + + +def _synthetic_catalogue(): + """ngmix-style catalogue with known responses and zero selection response.""" + rng = np.random.default_rng(1) + names = ["1M", "1P", "2M", "2P", "NOSHEAR"] + fields = [] + for name in names: + fields.extend( + [ + (f"NGMIX_FLAGS_{name}", "i4"), + (f"NGMIX_G1_{name}", "f8"), + (f"NGMIX_G2_{name}", "f8"), + (f"NGMIX_FLUX_{name}", "f8"), + (f"NGMIX_FLUX_ERR_{name}", "f8"), + (f"NGMIX_T_{name}", "f8"), + (f"NGMIX_T_ERR_{name}", "f8"), + (f"NGMIX_T_PSF_RECONV_{name}", "f8"), + ] + ) + fields += [("NGMIX_G1_ERR_NOSHEAR", "f8"), ("NGMIX_G2_ERR_NOSHEAR", "f8")] + ntot = N_PER_POP * len(POPULATIONS) + data = np.zeros(ntot, dtype=fields) + response = np.concatenate([np.full(N_PER_POP, r) for r, _ in POPULATIONS]) + w_des = np.concatenate([np.full(N_PER_POP, w) for _, w in POPULATIONS]) + intrinsic = np.empty((2, ntot)) + for k in range(len(POPULATIONS)): + half = rng.normal(0, 0.25, size=(2, N_PER_POP // 2)) + # Exact second moments give the target 1:3 weights in distinct cells. + r, target_w = POPULATIONS[k] + noise = r**2 / target_w - 0.5 * r**2 * np.sum(GAMMA**2) + half *= np.sqrt(noise / np.mean(half**2)) + intrinsic[:, k * N_PER_POP : (k + 1) * N_PER_POP] = np.concatenate( + [half, -half], axis=1 + ) + g_ns = intrinsic + response * GAMMA[:, None] + shifts = { + "NOSHEAR": (0.0, 0.0), + "1P": (STEP, 0.0), + "1M": (-STEP, 0.0), + "2P": (0.0, STEP), + "2M": (0.0, -STEP), + } + for name, (d1, d2) in shifts.items(): + data[f"NGMIX_G1_{name}"] = g_ns[0] + response * d1 + data[f"NGMIX_G2_{name}"] = g_ns[1] + response * d2 + data[f"NGMIX_FLUX_{name}"] = np.repeat([30.0, 100.0], N_PER_POP) + data[f"NGMIX_FLUX_ERR_{name}"] = 1.0 # SNR 50: inside all cuts. + data[f"NGMIX_T_{name}"] = 1.5 + data[f"NGMIX_T_PSF_RECONV_{name}"] = 1.0 # Size ratio inside all cuts. + data["NGMIX_G1_ERR_NOSHEAR"] = 0.01 + data["NGMIX_G2_ERR_NOSHEAR"] = 0.01 + return data, w_des + + +def test_w_des_weighted_shear_unbiased_with_committed_global_response_weight(): + """Protect multiplicative calibration of production w_des statistics. + + mask_v2.0.yaml sets the weight used to average the global response; + cosmo_val uses w_des with shear.R=1. A weighted shear estimator is unbiased + only when the applied response is averaged with the statistic's weights. + Intrinsic shapes cancel in pairs and selection response is zero here. + Responses 0.6/0.9 with w_des=1/3 give _w_des=0.825, while equal + measurement errors give _w_iv=0.75. The calibrated weighted mean must + equal gamma, not 1.1*gamma. + """ + root = _repository_root() + config = yaml.safe_load((root / "config/calibration/mask_v2.0.yaml").read_text())[ + "metacal" + ] + stat_config = yaml.safe_load((root / "cosmo_val/cat_config.yaml").read_text()) + # Check the production statistic's weight, rather than an unused template. + shear_config = stat_config["SP_v1.4.6.3"]["shear"] + assert shear_config["w_col"] == "w_des" + assert shear_config["R"] == 1.0 + data, w_des = _synthetic_catalogue() + gal = metacal( + data, + np.ones(len(data), bool), + snr_min=config["gal_snr_min"], + snr_max=config["gal_snr_max"], + rel_size_min=config["gal_rel_size_min"], + rel_size_max=config["gal_rel_size_max"], + size_corr_ell=config["gal_size_corr_ell"], + sigma_eps=config["sigma_eps_prior"], + weight_scheme=config["weight_scheme"], + ) + # The production W fit/evaluation must reproduce the target weights. + np.testing.assert_allclose(gal.ns["w_des"], w_des, rtol=1e-12) + np.testing.assert_allclose(np.diag(gal.R), 0.825, atol=1e-12) + assert len(gal.mask_dict["ns"]) == len(data) + np.testing.assert_allclose(gal.R_selection, 0, atol=1e-12) + g_corr, _, _, mask, _, _ = calibration.get_calibrated_m_c( + gal, additive_correction=False + ) + gamma_hat = np.array([np.average(g_corr[c], weights=w_des[mask]) for c in (0, 1)]) + assert np.allclose(gamma_hat, GAMMA, rtol=1e-6), ( + f"w_des-weighted shear biased: gamma_hat={gamma_hat}, gamma={GAMMA}, " + f"m={gamma_hat / GAMMA - 1}; global_R_weight=" + f"{config['weight_scheme']!r}, applied diag={np.diag(gal.R)}, " + "expected statistic response=0.825" + ) diff --git a/src/sp_validation/tests/test_consistent_response.py b/src/sp_validation/tests/test_consistent_response.py new file mode 100644 index 000000000..bc4db7ebe --- /dev/null +++ b/src/sp_validation/tests/test_consistent_response.py @@ -0,0 +1,385 @@ +"""Analytic S1/case-d checks; no astronomical input required.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import calibration, io, response + + +def catalogue(n=80): + fields = [] + for b in ("NOSHEAR", "1P", "1M", "2P", "2M"): + fields += [ + (f"NGMIX_{k}_{b}", "f8") + for k in ( + "FLAGS", + "G1", + "G2", + "G1_ERR", + "G2_ERR", + "FLUX", + "FLUX_ERR", + "T", + "T_ERR", + "T_PSF_RECONV", + ) + ] + d = np.zeros(n, dtype=fields) + matrix = np.array([[0.8, 0.12], [-0.06, 0.9]]) + intrinsic = np.tile([[0.2, -0.2], [0.1, -0.1]], (1, n // 2)) + ce = matrix @ np.array([0.02, -0.01]) + np.array([0.004, -0.002]) + for b, shift in [ + ("NOSHEAR", [0, 0]), + ("1P", [0.01, 0]), + ("1M", [-0.01, 0]), + ("2P", [0, 0.01]), + ("2M", [0, -0.01]), + ]: + e = intrinsic + ce[:, None] + (matrix @ shift)[:, None] + for i in (1, 2): + d[f"NGMIX_G{i}_{b}"] = e[i - 1] + d[f"NGMIX_FLUX_{b}"] = np.repeat([30.0, 100.0], n // 2) + d[f"NGMIX_FLUX_ERR_{b}"] = 1 + d[f"NGMIX_T_{b}"] = 1 + d[f"NGMIX_T_PSF_RECONV_{b}"] = 1 + return d, matrix, ce + + +@pytest.mark.parametrize("scheme", ["des", "iv"]) +def test_full_matrix_c_units_and_sim_signal(scheme): + d, r, ce = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme=scheme, + size_corr_ell=False, + des_num_bins=2, + ) + np.testing.assert_allclose(gm.R, r, atol=1e-14) + np.testing.assert_allclose(gm.R_joint, r, atol=1e-14) + np.testing.assert_allclose(gm.c_e, ce, atol=1e-14) + np.testing.assert_allclose(gm.c_g, np.linalg.solve(r, ce), atol=1e-14) + sim, _, w, _, _, _ = calibration.get_calibrated_m_c(gm, False) + real, *_ = calibration.get_calibrated_m_c(gm, True) + np.testing.assert_allclose(np.average(real, weights=w, axis=1), 0, atol=1e-14) + np.testing.assert_allclose(np.average(sim, weights=w, axis=1), gm.c_g) + + +def test_fixed_weight_c_uses_final_weights(): + d, _, _ = catalogue() + d["NGMIX_G1_ERR_NOSHEAR"] = np.linspace(0.01, 0.8, len(d)) + gm = calibration.metacal( + d, np.ones(len(d), bool), weight_scheme="iv", size_corr_ell=False + ) + _, raw, w, _, ce, _ = calibration.get_calibrated_m_c(gm) + np.testing.assert_allclose(ce, np.average(raw, weights=w, axis=1)) + assert abs(ce[0] - raw[0].mean()) > 1e-5 + + +def fixture_response(): + e = np.array([[0.2, -0.1, 0.4, -0.2], [0.1, 0.3, -0.2, 0.2]]) + r = np.array([[0.8, 0.1], [-0.04, 0.9]]) + branches = {"ns": e} + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + branches[p] = e + 0.01 * r[:, j, None] + branches[m] = e - 0.01 * r[:, j, None] + masks = {b: np.arange(4) for b in branches} + rg = np.repeat(r[:, :, None], 4, axis=2) + w = {b: np.array([1.0, 2.0, 3.0, 4.0]) for b in branches} + return branches, masks, rg, w + + +def test_selection_separate_denominators(): + b, masks, rg, w = fixture_response() + masks["p1"] = np.array([0, 1, 2]) + masks["m1"] = np.array([1, 3]) + res = response.responses(b, masks, w, rg, 0.01) + expected = ( + np.average(b["ns"][:, masks["p1"]], weights=w["p1"][masks["p1"]], axis=1) + - np.average(b["ns"][:, masks["m1"]], weights=w["m1"][masks["m1"]], axis=1) + ) / 0.02 + np.testing.assert_allclose(res["selection"][:, 0], expected) + + +def test_pure_cell_migration_all_elements(): + b, masks, rg, _ = fixture_response() + # Fixed two-cell W. No cut crossing; all four selection elements nonzero. + table = ( + np.array([10.0, 50.0, 500.0]), + np.array([0.5, 3.0]), + np.array([[1.0], [4.0]]), + None, + ) + positions = { + "ns": [20, 20, 100, 100], + "p1": [100, 20, 100, 20], + "m1": [20, 100, 20, 100], + "p2": [100, 100, 20, 100], + "m2": [20, 20, 100, 20], + } + w = { + k: response.evaluate_w(table, np.array(s), np.ones(4)) + for k, s in positions.items() + } + res = response.responses(b, masks, w, rg, 0.01) + for j, (p, m) in enumerate((("p1", "m1"), ("p2", "m2"))): + expected = ( + np.average(b["ns"], weights=w[p], axis=1) + - np.average(b["ns"], weights=w[m], axis=1) + ) / 0.02 + np.testing.assert_allclose(res["selection"][:, j], expected) + frozen = response.responses(b, masks, {k: w["ns"] for k in w}, rg, 0.01) + np.testing.assert_allclose(frozen["selection"], 0, atol=1e-14) + assert np.all(abs(res["selection"]) > 0.1) + np.testing.assert_allclose(res["joint"], res["total"], atol=1e-14) + scaled = response.responses(b, masks, {k: 17 * v for k, v in w.items()}, rg, 0.01) + np.testing.assert_allclose(scaled["total"], res["total"]) + + +def test_w_moments_edges_empty_invalid_and_new_rows(): + table = response.fit_w( + np.array([10.0, 50.0, 100.0]), + np.ones(3), + np.array([[0.2, 0.2, 0.4], [0.2, 0.2, 0.4]]), + np.array([0.8, 0.8, 0.8]), + np.array([10.0, 50.0, 500.0]), + np.array([0.5, 2.0, 3.0]), + ) + np.testing.assert_allclose(table[2], [[16.0, 0], [4.0, 0]]) + s = np.array([1.0, 50.0, 51.0, 1000.0, np.nan, 20.0]) + size = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 2.5]) + np.testing.assert_allclose( + response.evaluate_w(table, s, size), [16, 16, 4, 4, 0, 0] + ) + + +def test_spatial_jackknife_shared_selection_and_covariance(): + b, masks, rg, w = fixture_response() + groups = np.array(["a", "b", "c", "d"]) + masks["p1"] = np.array([0, 1, 2]) + masks["m1"] = np.array([1, 2, 3]) + labels, estimates, cov = response.jackknife(b, masks, w, rg, 0.01, groups) + brute = [response.responses(b, masks, w, rg, 0.01, groups != g) for g in labels] + for key in estimates: + np.testing.assert_allclose(estimates[key], [x[key] for x in brute]) + v = np.concatenate([estimates[k].reshape(4, -1) for k in estimates], axis=1) + dv = v - v.mean(axis=0) + np.testing.assert_allclose(cov, 0.75 * dv.T @ dv) + assert cov[4, 4] > 0 + + +def test_end_to_end_cell_migration_and_sheared_only_row(): + d, _, _ = catalogue() + for branch in ("NOSHEAR", "1P", "1M", "2P", "2M"): + d[f"NGMIX_G1_{branch}"][40:] += np.tile([0.15, -0.15], 20) + d["NGMIX_FLUX_1P"][:40:2] = 100 + d["NGMIX_FLUX_1M"][1:40:2] = 100 + # A common-parent row absent from S_ns still gets a defined branch W. + d["NGMIX_FLUX_NOSHEAR"][0] = 9 + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + assert 0 not in gm.mask_dict["ns"] + assert 0 in gm.mask_dict["p1"] + assert gm.branch_weights["p1"][0] > 0 + assert gm.branch_migration["p1"] > 0 + manual = [] + for key in ("p1", "m1"): + idx = gm.mask_dict[key] + manual.append( + np.average( + np.array([gm.ns["g1"][idx], gm.ns["g2"][idx]]), + weights=gm.branch_weights[key][idx], + axis=1, + ) + ) + np.testing.assert_allclose(gm.R_selection[:, 0], (manual[0] - manual[1]) / 0.02) + frozen = {b: gm.branch_weights["ns"] for b in response.BRANCHES} + res = response.responses(gm.branches, gm.mask_dict, frozen, gm.R_shear, 0.01) + assert abs(res["selection"][0, 0] - gm.R_selection[0, 0]) > 0.1 + + +def test_coherent_spatial_errors_exceed_independent_object_errors(): + d, _, _ = catalogue() + groups = np.repeat(np.arange(4), 20) + for comp in (1, 2): + for sign, step in (("P", 0.01), ("M", -0.01)): + d[f"NGMIX_G{comp}_{comp}{sign}"] += step * groups * 0.1 + gm = calibration.metacal( + d, np.ones(len(d), bool), weight_scheme="iv", size_corr_ell=False + ) + _, _, spatial = gm.response_jackknife(groups) + _, _, objects = gm.response_jackknife(np.arange(len(d))) + assert spatial[0, 0] > 10 * objects[0, 0] + + +def test_mismatched_config_and_recalibration_refusal(tmp_path): + d, _, _ = catalogue() + with pytest.raises(ValueError, match="disagrees"): + calibration.metacal(d, np.ones(len(d), bool), weight_scheme="iv", w_col="w_des") + hdr = fits.Header({"R_S11": 0.1, "R_S12": 0.0, "R_S21": 0.0, "R_S22": 0.1}) + path = tmp_path / "insufficient.fits" + fits.HDUList([fits.PrimaryHDU(header=hdr), fits.BinTableHDU()]).writeto(path) + with pytest.raises(ValueError, match="parent five-branch"): + calibration.get_calibrate_e_from_cat(path) + + +def test_recalibration_requires_unchanged_population_with_nonzero_selection(tmp_path): + path = tmp_path / "compatible.fits" + truth = np.repeat([0.02, -0.02], 4) + raw = 0.9 * truth + zeros = np.zeros(8) + cols = [ + fits.Column(name="e1_uncal", array=raw, format="D"), + fits.Column(name="e2_uncal", array=zeros, format="D"), + fits.Column(name="w_des", array=np.ones(8), format="D"), + ] + for i in (1, 2): + for j in (1, 2): + cols.append( + fits.Column( + name=f"R_g{i}{j}", + array=np.full(8, 0.8 if i == j else 0), + format="D", + ) + ) + hdr = fits.Header( + { + "R_S11": 0.1, + "R_S12": 0.0, + "R_S21": 0.0, + "R_S22": 0.1, + "WGHTSCH": "des", + "N_ns": 8, + "SW_ns": 8.0, + } + ) + fits.HDUList( + [fits.PrimaryHDU(header=hdr), fits.BinTableHDU.from_columns(cols)] + ).writeto(path) + g1, _ = calibration.get_calibrate_e_from_cat(path) + np.testing.assert_allclose(g1, truth) + with fits.open(path, mode="update") as hdus: + hdus[0].header["N_ns"] = 9 + with pytest.raises(ValueError, match="population/weights"): + calibration.get_calibrate_e_from_cat(path) + + +def test_downstream_c_is_calibrated_not_raw(): + from types import SimpleNamespace + + from sp_validation.cosmo_val.core import CosmologyValidation + + e = np.array([[0.12, -0.04], [0.08, 0.16]]) + cg = np.array([0.02, -0.03]) + obj = SimpleNamespace( + _read_shear_cols=lambda *args: e, + _component_responses=lambda ver: (0.8, 0.8), + cc={"external": {"shear": {"R": 0.8}}}, + c1={"external": cg[0]}, + c2={"external": cg[1]}, + ) + result = CosmologyValidation._calibrated_g(obj, "external") + np.testing.assert_allclose(result, e / 0.8 - cg[:, None]) + assert not np.allclose(result, (e - cg[:, None]) / 0.8) + + +@pytest.mark.parametrize("adapter", ["xi", "pseudo_cl", "rho_tau", "leakage"]) +def test_calibrated_matrix_shear_passes_reader_adapters_once( + tmp_path, monkeypatch, adapter +): + from types import SimpleNamespace + + from sp_validation.cosmo_val import core + from sp_validation.rho_tau import _CatalogueLoader + + d, _, _ = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + g, _, w, _, _, _ = calibration.get_calibrated_m_c(gm, False) + path = tmp_path / "matrix_calibrated.fits" + cols = [ + fits.Column(name="e1", array=g[0], format="D"), + fits.Column(name="e2", array=g[1], format="D"), + fits.Column(name="w_des", array=w, format="D"), + ] + fits.HDUList([fits.PrimaryHDU(), fits.BinTableHDU.from_columns(cols)]).writeto(path) + calibration.save_response_product(path, gm) + entry = dict(path=path, w_col="w_des", e1_col="e1", e2_col="e2", R=1.0) + + def read(block): + if adapter == "rho_tau": + loader = _CatalogueLoader({"shear": block}, {}) + filename = loader("shear") # ordinary FITS is passed through + with fits.open(filename) as hdus: + return np.array([hdus[1].data["e1"], hdus[1].data["e2"]]) + if adapter == "leakage": + obj = core._LeakageObject.__new__(core._LeakageObject) + obj.entries = {"shear": block} + obj.read_data() + return np.array([obj._dat["e1"], obj._dat["e2"]]) + if adapter == "xi": + monkeypatch.setattr(core.leakage, "cut_data", lambda data, *args: data) + obj = core._LeakageScale.__new__(core._LeakageScale) + obj.entries = {"shear": block} + obj._params = {"cut": None, "verbose": False} + obj.read_data(shear=True, psf=False) + return np.array([obj.dat_shear["e1"], obj.dat_shear["e2"]]) + obj = SimpleNamespace( + cc={"SP": {"shear": block}}, + results={"SP": SimpleNamespace(dat_shear=io.open_entry(block))}, + ) + return np.array( + core.CosmologyValidation._read_shear_cols(obj, "SP", "e1_col", "e2_col") + ) + + np.testing.assert_allclose(read(entry), g) + for wrong in ( + dict(entry, w_col="w_iv"), + dict(entry, R=0.8), + dict(entry, R11="R_g11"), + ): + with pytest.raises(ValueError): + read(wrong) + + +def test_persistence_and_downstream_no_double_R(tmp_path): + d, _, _ = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + path = tmp_path / "cut.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.BinTableHDU.from_columns( + [fits.Column(name="w_des", array=gm.ns["w"], format="D")] + ), + ] + ).writeto(path) + calibration.save_response_product(path, gm, np.arange(len(d)) % 4) + with fits.open(path) as hdus: + assert hdus["R_COV"].data.shape == (20, 20) + assert hdus[0].header["RAPPLIED"] + assert hdus[0].header["WGHTSCH"] == "des" + for entry in ( + {"path": path, "w_col": "w_iv", "R": 1}, + {"path": path, "w_col": "w_des", "R": 0.8}, + ): + with pytest.raises(ValueError): + io.open_entry(entry) + assert len(io.open_entry({"path": path, "w_col": "w_des", "R": 1})) == len(d) diff --git a/src/sp_validation/tests/test_des_jackknife.py b/src/sp_validation/tests/test_des_jackknife.py new file mode 100644 index 000000000..c19898d74 --- /dev/null +++ b/src/sp_validation/tests/test_des_jackknife.py @@ -0,0 +1,81 @@ +"""Refitted cell sums versus the retained-row reference, including empty cells.""" + +import numpy as np +import pytest + +from sp_validation import calibration, response +from sp_validation.des_jackknife import accumulate +from sp_validation.tests.test_consistent_response import catalogue + + +@pytest.mark.parametrize("min_count", [0, 3, 10, 1000]) +def test_refit_cell_sums_match_rescans_with_empty_cell(min_count): + data, _, _ = catalogue() + # Only group zero trains the upper-right cell. Other artificial branches + # migrate into it, so its newly pooled weight affects the retained response. + for suffix in ("NOSHEAR", "1P", "1M", "2P", "2M"): + data[f"NGMIX_FLUX_{suffix}"] = 30 + data[f"NGMIX_T_{suffix}"] = 0.7 + data[f"NGMIX_FLUX_{suffix}"][:3] = 200 + data[f"NGMIX_T_{suffix}"][:3] = 2 + data[f"NGMIX_G1_{suffix}"][:20] += 0.13 + data["NGMIX_FLUX_1P"][20:24] = 200 + data["NGMIX_T_1P"][20:24] = 2 + data["NGMIX_FLUX_2M"][30:35] = 200 + data["NGMIX_T_2M"][30:35] = 2 + # Sheared-only and no-shear-only objects exercise mask/rg alignment. + data["NGMIX_FLUX_NOSHEAR"][50] = 9 + data["NGMIX_FLUX_1M"][51] = 9 + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + des_min_count=min_count, + ) + groups = np.repeat(np.arange(4), 20) + acc = accumulate(gm, groups, chunk_size=7) + assert gm.W[3][1, 1] == 3 + assert acc.training[0, :, 3].sum() - acc.training[0, 0, 3] == 0 + for k, label in enumerate(acc.labels): + keep = groups != label + idx = gm.mask_dict["ns"] + retained = keep[idx] + table = response.fit_w( + (gm.ns["flux"] / gm.ns["flux_err"])[idx][retained], + (gm.ns["T"] / gm.ns["Tpsf"])[idx][retained], + np.array(gm.branches["ns"])[:, idx][:, retained], + (0.5 * (gm.R11 + gm.R22))[retained], + gm.W[0], + gm.W[1], + min_count=min_count, + ) + np.testing.assert_allclose(acc.deleted_weight(k), table[2], atol=1e-12, rtol=0) + if k == 0: + assert (acc.deleted_weight(k)[1, 1] > 0) == (min_count > 0) + labels, fast, cov = gm.response_jackknife(groups, refit_w=True) + ref_labels, ref, ref_cov = gm.response_jackknife( + groups, refit_w=True, reference=True + ) + np.testing.assert_array_equal(labels, ref_labels) + for key in fast: + np.testing.assert_allclose(fast[key], ref[key], atol=1e-12, rtol=0) + np.testing.assert_allclose( + acc.jackknife()[1][key], ref[key], atol=1e-12, rtol=0 + ) + np.testing.assert_allclose(cov, ref_cov, atol=1e-12, rtol=0) + + +def test_refit_group_validation(): + data, _, _ = catalogue() + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + ) + with pytest.raises(ValueError, match="align"): + gm.response_jackknife(np.arange(10), refit_w=True) + with pytest.raises(ValueError, match="at least two"): + gm.response_jackknife(np.zeros(len(data)), refit_w=True) diff --git a/src/sp_validation/tests/test_des_table_independence.py b/src/sp_validation/tests/test_des_table_independence.py new file mode 100644 index 000000000..1087d235c --- /dev/null +++ b/src/sp_validation/tests/test_des_table_independence.py @@ -0,0 +1,45 @@ +"""A DES table fitted on the measured population suppresses its shear signal.""" + +from itertools import product + +import numpy as np +import pytest + +from sp_validation import response + + +@pytest.mark.parametrize("occupancy", [3, 4, 8]) +def test_self_fitted_table_bias_is_inverse_cell_occupancy(occupancy): + # Enumerate every Rademacher sample instead of drawing Monte Carlo trials. + # Each sample is one cell; duplicate it for two independent training signs. + signs = np.array(list(product((-1.0, 1.0), repeat=occupancy))) + x = np.repeat(signs, 2, axis=0) * 0.25 + e = np.stack((x, np.full_like(x, 0.25))) + independent = e.copy() + independent[0] = np.tile([-0.25, 0.25], len(signs))[:, None] + cells = len(x) + snr = np.repeat(np.arange(cells) + 0.5, occupancy) + size = np.ones(len(snr)) + edges_s = np.arange(cells + 1) + edges_r = np.array([0.0, 2.0]) + step = 1e-5 + biases = [] + for training in (e, independent): + means = [] + for shear in (-step, step): + shifted = training.copy() + shifted[0] += shear + table = response.fit_w( + snr, + size, + shifted.reshape(2, -1), + np.ones(len(snr)), + edges_s, + edges_r, + min_count=3, + ) + weights = response.evaluate_w(table, snr, size) + means.append(np.average((e[0] + shear).ravel(), weights=weights)) + biases.append((means[1] - means[0]) / (2 * step) - 1) + assert biases[0] == pytest.approx(-1 / occupancy, abs=3e-9) + assert biases[1] == pytest.approx(0, abs=3e-9) diff --git a/src/sp_validation/tests/test_downstream_calibrated_shear.py b/src/sp_validation/tests/test_downstream_calibrated_shear.py new file mode 100644 index 000000000..2d2a698fb --- /dev/null +++ b/src/sp_validation/tests/test_downstream_calibrated_shear.py @@ -0,0 +1,37 @@ +"""Real- and harmonic-space estimators share calibrated shear.""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +from sp_validation.cosmo_val import pseudo_cl +from sp_validation.cosmo_val.core import CosmologyValidation + + +@pytest.mark.parametrize("response,mean_subtracted", [(0.5, False), (1.0, True)]) +def test_pseudo_cl_uses_xi_shear(monkeypatch, response, mean_subtracted): + catalog = np.array( + [(1.0, 2.0, 1.0), (3.0, -1.0, 3.0)], + dtype=[("e1", "f8"), ("e2", "f8"), ("w", "f8")], + ) + if mean_subtracted: + for col in ("e1", "e2"): + catalog[col] -= np.average(catalog[col], weights=catalog["w"]) + obj = CosmologyValidation.__new__(CosmologyValidation) + obj.cc = { + "test": {"shear": {"e1_col": "e1", "e2_col": "e2", "w_col": "w", "R": response}} + } + obj._results = {"test": SimpleNamespace(dat_shear=catalog)} + obj._c1 = {"test": np.average(catalog["e1"] / response, weights=catalog["w"])} + obj._c2 = {"test": np.average(catalog["e2"] / response, weights=catalog["w"])} + monkeypatch.setattr(pseudo_cl, "open_entry", lambda entry: catalog) + xi = obj._calibrated_g("test") + harmonic = obj._pseudo_cl_catalogue("test") + np.testing.assert_allclose(harmonic["e1"], xi[0]) + np.testing.assert_allclose(harmonic["e2"], xi[1]) + if mean_subtracted: + np.testing.assert_allclose(harmonic["e1"], catalog["e1"]) + np.testing.assert_allclose(harmonic["e2"], catalog["e2"]) + else: + np.testing.assert_allclose(xi[0], [-3.0, 1.0]) diff --git a/src/sp_validation/tests/test_external_des.py b/src/sp_validation/tests/test_external_des.py new file mode 100644 index 000000000..92112342b --- /dev/null +++ b/src/sp_validation/tests/test_external_des.py @@ -0,0 +1,126 @@ +"""Independent data-table policy for image-simulation calibration.""" + +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import yaml +from astropy.io import fits + +from sp_validation import calibration, catalog_builders, response +from sp_validation.image_sim_config import calibration_config +from sp_validation.tests.test_consistent_response import catalogue + + +def make_product(tmp_path): + data, _, _ = catalogue() + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + path = tmp_path / "data-calibration.fits" + fits.HDUList([fits.PrimaryHDU()]).writeto(path) + calibration.save_response_product(path, gm) + return dict(name="synthetic-data-calibration", path=str(path)), gm + + +def test_external_table_never_trains_on_sim_shapes_and_uses_fixed_jk(tmp_path): + product, source = make_product(tmp_path) + data, _, _ = catalogue() + for suffix in ("NOSHEAR", "1P", "1M", "2P", "2M"): + data[f"NGMIX_G1_{suffix}"][:20] += 0.3 + kwargs = dict(weight_scheme="des", size_corr_ell=False, des_weight_product=product) + gm = calibration.metacal(data, np.ones(len(data), bool), **kwargs) + # Product edges win, even when the sim's configured bin count differs. + np.testing.assert_array_equal(gm.W[2], source.W[2]) + for b in response.BRANCHES: + np.testing.assert_array_equal(gm.branch_weights[b], source.branch_weights[b]) + groups = np.repeat(np.arange(4), 20) + labels, jk, cov = gm.response_jackknife(groups, refit_w=True) + fixed = gm.response_jackknife(groups, refit_w=False) + np.testing.assert_allclose(cov, fixed[2]) + for key in jk: + np.testing.assert_allclose(jk[key], fixed[1][key]) + for j, label in enumerate(labels): + keep = groups != label + retained = calibration.metacal(data[keep], np.ones(keep.sum(), bool), **kwargs) + np.testing.assert_allclose(jk["c_g"][j], retained.c_g, atol=1e-13) + assert gm.refit_weights(keep) is gm.branch_weights + path = tmp_path / "sim.fits" + header = fits.Header() + # Exercise the catalogue driver's config-to-header path with the named + # product mapping, before attaching the response and weight extensions. + catalog_builders.CalibrateCat().add_params_to_FITS_header( + header, cm=dict(weight_scheme="des", des_weight_product=product) + ) + fits.HDUList([fits.PrimaryHDU(header=header)]).writeto(path) + calibration.save_response_product(path, gm, groups) + with fits.open(path) as f: + assert f[0].header["WPOLICY"] == "fixed-external" + assert f[0].header["WSOURCE"] == "external" + assert f[0].header["WPRODUCT"] == product["name"] + np.testing.assert_allclose(f["R_JK"].data["c_g1"], jk["c_g"][:, 0]) + np.testing.assert_array_equal(f["W_DES"].data, source.W[2]) + + +def test_reject_product_without_verified_coordinates(tmp_path): + product, _ = make_product(tmp_path) + with fits.open(product["path"], mode="update") as f: + del f[0].header["WSIZDEF"] + with pytest.raises(ValueError, match="coordinate metadata"): + calibration.load_des_weight_product(product) + + +def test_staging_replaces_old_symlink_without_editing_shared_config(tmp_path): + original = "metacal:\n global_R_weight: null\n additive_correction: true\n" + source = tmp_path / "mask.yaml" + source.write_text(original) + staged = tmp_path / "config_mask.yaml" + staged.symlink_to(source) + script = ( + Path(__file__).resolve().parents[3] + / "scripts/calibration/stage_image_sim_config.py" + ) + subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(source), + "--output", + str(staged), + "--w-cols", + "w_des", + "--product-name", + "data", + "--product-path", + "/data.fits", + ], + check=True, + ) + assert not staged.is_symlink() + assert source.read_text() == original + cm = yaml.safe_load(staged.read_text())["metacal"] + assert cm["des_weight_product"] == dict(name="data", path="/data.fits") + assert cm["weight_scheme"] == "des" + + +def test_image_sim_default_requires_named_external_product(): + original = dict(metacal=dict(global_R_weight=None, additive_correction=True)) + with pytest.raises(ValueError, match="require des_weight_product"): + calibration_config(original, ["w_des"]) + product = dict(name="data-release", path="/calibration/data.fits") + staged = calibration_config(original, ["w_des"], product=product) + assert staged["metacal"]["des_weight_product"] == product + assert staged["metacal"]["weight_scheme"] == "des" + assert not staged["metacal"]["additive_correction"] + assert "global_R_weight" not in staged["metacal"] + assert "weight_scheme" not in original["metacal"] + diagnostic = calibration_config(staged, ["w_des"], source="self") + assert "des_weight_product" not in diagnostic["metacal"] + assert calibration_config(original, ["w_iv"]) == original diff --git a/src/sp_validation/tests/test_image_sims_jackknife.py b/src/sp_validation/tests/test_image_sims_jackknife.py new file mode 100644 index 000000000..171f260cb --- /dev/null +++ b/src/sp_validation/tests/test_image_sims_jackknife.py @@ -0,0 +1,100 @@ +"""Shared-tile, response-recalibrating m/c covariance.""" + +import numpy as np +import pytest + +from sp_validation.image_sims import ImageSimMBias + + +@pytest.mark.parametrize("refit", [False, True]) +def test_shared_tile_deletions_recalibrate_both_branches(refit): + estimator = ImageSimMBias( + dict( + shear_amplitude=0.025, + match_radius_deg=0.0002, + pair_match=False, + w_cols=["w_des"], + n_bootstrap=5, + bootstrap_seed=42, + ) + ) + tiles = np.array(["a", "a", "b", "c"]) + labels = np.array(["c", "a", "b"]) + expected = [] + fixed_response = [] + for p, m, comp in estimator.pairs: + for name, sign in ((p, 1), (m, -1)): + raw = np.array([[0.1, -0.2, 0.03, 0.2], [0.2, -0.1, 0.3, -0.2]]) + raw[comp] += sign * 0.025 + # Off-diagonal terms exercise matrix orientation and coupling. + response = np.array([[0.8, 0.03], [-0.02, 0.7]]) + deleted_R = np.array( + [response + np.eye(2) * sign * x for x in (0.04, -0.02, 0.01)] + ) + g = np.linalg.solve(response, raw) + estimator.cats[name] = dict( + raw=raw, + e1=g[0], + e2=g[1], + tiles=tiles, + jk_labels=labels.copy(), + jk_R=deleted_R, + w={"w_des": np.array([1.0, 2.0, 3.0, 4.0])}, + weight_scheme="des", + W_policy="refit" if refit else "fixed", + jk_g=np.array( + [ + np.linalg.solve( + deleted_R[j], + np.average( + raw[:, tiles != label], + axis=1, + weights=np.array([1.0, 2.0, 3.0, 4.0])[tiles != label] + ** 2, + ), + ) + for j, label in enumerate(labels) + ] + ), + ) + draws, held_R = [], [] + for label in sorted(labels): + means, fixed = [], [] + for name in (p, m): + cat = estimator.cats[name] + keep = tiles != label + mean = np.average( + cat["raw"][:, keep], axis=1, weights=cat["w"]["w_des"][keep] + ) + index = list(labels).index(label) + means.append( + cat["jk_g"][index, comp] + if refit + else np.linalg.solve(cat["jk_R"][index], mean)[comp] + ) + fixed.append( + np.average( + cat[f"e{comp + 1}"][keep], weights=cat["w"]["w_des"][keep] + ) + ) + draws.append([(means[0] - means[1]) / 0.05 - 1, (means[0] + means[1]) / 2]) + held_R.append((fixed[0] - fixed[1]) / 0.05 - 1) + expected.extend(np.array(draws).T) + fixed_response.append(held_R) + result = estimator.run(False) + expected = np.array(expected).T + delta = expected - expected.mean(axis=0) + covariance = 2 / 3 * delta.T @ delta + np.testing.assert_allclose(result["mc_jackknife"], expected) + np.testing.assert_allclose(result["mc_covariance"], covariance) + np.testing.assert_allclose( + [result[k + "_err"] for k in result["covariance_order"]], + np.sqrt(covariance.diagonal()), + ) + assert abs(covariance[0, 1]) > 1e-6 + assert not np.allclose(expected[:, 0], fixed_response[0]) + assert result["uncertainty"] == "shared_tile_response_jackknife" + assert "conditional_object_bootstrap_fixed_response" not in result + estimator.cats[estimator.pairs[0][1]]["jk_labels"][0] = "d" + with pytest.raises(ValueError, match="identical parent"): + estimator.run(False) diff --git a/src/sp_validation/tests/test_image_sims_weights.py b/src/sp_validation/tests/test_image_sims_weights.py new file mode 100644 index 000000000..b2ba74dfa --- /dev/null +++ b/src/sp_validation/tests/test_image_sims_weights.py @@ -0,0 +1,67 @@ +"""Image-simulation numerator weights and paired-estimator restrictions.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation.image_sims import ImageSimMBias + + +def test_modern_headline_uses_branch_weights(tmp_path): + """Symmetric weights remove a real numerator contribution on matched rows.""" + for branch, sign in (("plus", 1), ("minus", -1)): + directory = tmp_path / f"{branch}_grid_1" + directory.mkdir() + arrays = dict( + RA=[30.0, 31.0], + Dec=[0.0, 0.0], + e1=np.array([-0.1, 0.1]) + sign * 0.02, + e2=[0.0, 0.0], + w_des=[1.0, 3.0] if sign == 1 else [3.0, 1.0], + ) + arrays["e1_uncal"] = arrays["e1"] + arrays["e2_uncal"] = arrays["e2"] + table = fits.BinTableHDU.from_columns( + [fits.Column(name=k, array=v, format="D") for k, v in arrays.items()] + + [fits.Column(name="TILE_ID", array=["a", "b"], format="1A")] + ) + jk = fits.BinTableHDU.from_columns( + [ + fits.Column(name="TILE_ID", array=["b", "a"], format="1A"), + *[ + fits.Column(name=f"R_{i}{j}", array=[float(i == j)] * 2, format="D") + for i in (1, 2) + for j in (1, 2) + ], + ], + name="R_JK", + ) + primary = fits.PrimaryHDU() + primary.header["WGHTSCH"] = "des" + primary.header["RAPPLIED"] = True + fits.HDUList([primary, table, jk]).writeto(directory / "cat.fits") + cfg = dict( + grids_dir=str(tmp_path), + num=1, + catalog_name="cat.fits", + shear_amplitude=0.02, + match_radius_deg=0.0002, + w_cols=["w_des"], + n_bootstrap=10, + bootstrap_seed=42, + pair_match=False, + branches=["plus", "minus"], + pairs=[dict(plus="plus", minus="minus", component=0)], + ) + estimator = ImageSimMBias(cfg) + estimator.load_catalogs(False) + result = estimator.run(False) + assert result["m1"] == pytest.approx(2.5) + # All objects match, but symmetric weighting would yield m=0. + symmetric_m = ( + np.mean((estimator.cats["plus"]["e1"] - estimator.cats["minus"]["e1"]) / 0.04) + - 1 + ) + assert symmetric_m == pytest.approx(0) + with pytest.raises(ValueError, match="pair_match=False"): + ImageSimMBias({**cfg, "pair_match": True}).load_catalogs(False) diff --git a/src/sp_validation/tests/test_inverse_variance.py b/src/sp_validation/tests/test_inverse_variance.py new file mode 100644 index 000000000..97054f48a --- /dev/null +++ b/src/sp_validation/tests/test_inverse_variance.py @@ -0,0 +1,87 @@ +"""ShapePipe G*_ERR columns store standard deviations, not variances.""" + +import numpy as np +import pytest + +from sp_validation.calibration import metacal + + +def _catalogue(): + fields = [] + for shear in ("NOSHEAR", "1M", "1P", "2M", "2P"): + fields.extend( + (f"NGMIX_{name}_{shear}", "f8") + for name in ( + "FLAGS", + "G1", + "G2", + "G1_ERR", + "G2_ERR", + "FLUX", + "FLUX_ERR", + "T", + "T_ERR", + "T_PSF_RECONV", + ) + ) + data = np.zeros(2, dtype=fields) + for shear in ("NOSHEAR", "1M", "1P", "2M", "2P"): + data[f"NGMIX_FLUX_{shear}"] = 50.0 + data[f"NGMIX_FLUX_ERR_{shear}"] = 1.0 + data[f"NGMIX_T_{shear}"] = 1.0 + data[f"NGMIX_T_PSF_RECONV_{shear}"] = 1.0 + # ERR columns store sqrt(diag(g_cov)), not diag(g_cov). + for shear in ("NOSHEAR", "1M", "1P", "2M", "2P"): + data[f"NGMIX_G1_ERR_{shear}"] = [0.05, 0.20] + data[f"NGMIX_G2_ERR_{shear}"] = [0.10, 0.30] + # Exactly diagonal per-object shear responses 0.6 I and 1.0 I. + for component in (1, 2): + data[f"NGMIX_G{component}_{component}P"] = [0.006, 0.010] + data[f"NGMIX_G{component}_{component}M"] = [-0.006, -0.010] + return data + + +@pytest.mark.parametrize("mask", [None, np.array([False, True])], ids=["all", "masked"]) +def test_iv_weights_square_both_shape_error_standard_deviations(mask): + """Protect weights from adding standard deviations to intrinsic variance. + + ShapePipe stores G1_ERR/G2_ERR as square roots of covariance diagonals. + For sigma_eps=0.34 and errors (0.05,0.10)/(0.20,0.30), the variances are + 2*0.34**2+0.05**2+0.10**2=0.2437 and + 2*0.34**2+0.20**2+0.30**2=0.3612. Their reciprocals are the correct + inverse-variance weights, including after masking. + """ + _, _, actual = metacal.get_variance_ivweights(_catalogue(), 0.34, mask=mask) + expected = 1.0 / np.array([0.2437, 0.3612]) + if mask is not None: + expected = expected[mask] + np.testing.assert_allclose(actual, expected, rtol=1e-12, atol=0) + + +def test_returned_covariance_diagonals_are_variances(): + c11, c22, _ = metacal.get_variance_ivweights(_catalogue(), 0.34) + np.testing.assert_allclose(c11, np.array([0.05, 0.20]) ** 2) + np.testing.assert_allclose(c22, np.array([0.10, 0.30]) ** 2) + + +def test_global_shear_response_uses_squared_shape_error_weights(): + """Protect the applied response from the same dimensional weight error. + + Both objects pass every sheared selection, so selection response is zero. + Their finite-difference shear responses are 0.6 I and 1.0 I. With + global_R_weight='w', the correct diagonal response is + (0.6/0.2437+1/0.3612)/(1/0.2437+1/0.3612), using intrinsic plus + measurement variances, rather than standard deviations. + """ + result = metacal( + _catalogue(), + np.ones(2, dtype=bool), + sigma_eps=0.34, + global_R_weight="w", + size_corr_ell=False, + ) + np.testing.assert_allclose(result.R_selection, np.zeros((2, 2)), atol=1e-14) + expected_scalar = (0.6 * 0.3612 + 1.0 * 0.2437) / (0.3612 + 0.2437) + np.testing.assert_allclose( + result.R, np.eye(2) * expected_scalar, rtol=1e-12, atol=1e-14 + ) diff --git a/src/sp_validation/tests/test_leakage_weight_scheme.py b/src/sp_validation/tests/test_leakage_weight_scheme.py new file mode 100644 index 000000000..bc5870d5a --- /dev/null +++ b/src/sp_validation/tests/test_leakage_weight_scheme.py @@ -0,0 +1,32 @@ +"""The leakage fit must use the science catalogue's weights.""" + +from types import SimpleNamespace + +import numpy as np + +from sp_validation import catalog_builders + + +def test_leakage_uses_iv_weights(monkeypatch): + cat = {"w_des": np.array([1.0, 0.0]), "w_iv": np.array([0.0, 1.0])} + data = {"e1_PSF": np.array([1.0, 2.0]), "e2_PSF": np.array([2.0, 3.0])} + monkeypatch.setattr( + catalog_builders.sp_cat, "get_col", lambda dat, col, *args: dat[col] + ) + + def fit(gal, num_bins, weight_type): + weights = gal[f"w_{weight_type}"] + return np.average(gal["e1"], weights=weights), 0.0 + + monkeypatch.setattr( + catalog_builders.calibration, "get_alpha_leakage_per_object", fit + ) + result = catalog_builders.compute_PSF_leakage( + cat, + (np.array([1.0, 3.0]), np.array([0.0, 0.0])), + data, + SimpleNamespace(_mask=None), + None, + weight_type="iv", + ) + assert result == (3.0, 0.0) diff --git a/src/sp_validation/tests/test_pair_response_jackknife.py b/src/sp_validation/tests/test_pair_response_jackknife.py new file mode 100644 index 000000000..37273701e --- /dev/null +++ b/src/sp_validation/tests/test_pair_response_jackknife.py @@ -0,0 +1,48 @@ +"""Stage-1 paired/global covariance, checked against explicit pair sums.""" + +import numpy as np +import pytest + +from sp_validation.response_diagnostics import scalar_pair_response + + +@pytest.mark.parametrize("varying", [False, True]) +def test_scalar_response_shared_patch_deletions(varying): + ra = np.arange(12, dtype=float) * 0.01 + dec = np.zeros(12) + w = np.arange(12, dtype=float) + 1 + r = np.linspace(0.5, 1.1, 12) if varying else np.full(12, 0.8) + patch = np.repeat(np.arange(4), 3) + measured = scalar_pair_response( + ra, + dec, + w, + r, + patch=patch, + min_sep=0.1, + max_sep=10, + nbins=1, + bin_slop=0, + ) + + def brute(keep): + ii, jj = np.triu_indices(len(ra), 1) + selected = keep[ii] & keep[jj] + ww = w[ii[selected]] * w[jj[selected]] + pair = np.average(r[ii[selected]] * r[jj[selected]], weights=ww) + return pair / np.average(r[keep], weights=w[keep]) ** 2 - 1 + + expected = np.array([brute(patch != j) for j in range(4)]) + # TreeCorr scalar fields are stored at float32 precision. + np.testing.assert_allclose( + measured["fraction"], brute(np.ones(12, bool)), atol=1e-7 + ) + np.testing.assert_allclose(measured["deleted"][:, 0], expected, atol=1e-7) + np.testing.assert_allclose( + measured["error"][0], + np.sqrt(0.75 * np.sum((expected - expected.mean()) ** 2)), + atol=1e-7, + ) + if not varying: + np.testing.assert_allclose(measured["fraction"], 0, atol=1e-7) + np.testing.assert_allclose(measured["error"], 0, atol=1e-7) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 929d3fb46..174cd6f2d 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -789,6 +789,12 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): def test__measure_pseudo_cl_catalog_end_to_end(cv, tmp_path, monkeypatch): """Catalog SACC output retains the unwindowed workspace bandpower window.""" ver = cv._test_version + shear = cv.cc[ver]["shear"] + monkeypatch.setattr( + cv, + "_calibrated_g", + lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]), + ) cv.cell_method = "catalog" cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} saved = {} @@ -945,7 +951,7 @@ def test_calculate_pseudo_cl_tomography_requires_one_nz_per_bin(cv): cv.calculate_pseudo_cl(compute_tomography=True) -def test__measure_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): +def test__measure_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path, monkeypatch): """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to @@ -953,6 +959,13 @@ def test__measure_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): (it drops the BE row); we pin the round-tripped table. """ ver = cv._test_version + shear = cv.cc[ver]["shear"] + # This round-trip regression pins the historical uncentered input. + monkeypatch.setattr( + cv, + "_calibrated_g", + lambda ver, cat: (cat[shear["e1_col"]], cat[shear["e2_col"]]), + ) cv._pseudo_cls = { ver: { "tomo_bin_1_tomo_bin_1": {}, diff --git a/src/sp_validation/tests/test_recalibration_weights.py b/src/sp_validation/tests/test_recalibration_weights.py new file mode 100644 index 000000000..84453bdab --- /dev/null +++ b/src/sp_validation/tests/test_recalibration_weights.py @@ -0,0 +1,73 @@ +"""Recalibration must use the statistic's weights for both response and bias.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation.calibration import get_calibrate_e_from_cat + +G_TRUE = 0.02 + + +def _write_catalogue(path, weight_type): + # Opposite weight preferences: (response, w_des, w_iv). + # Intrinsic +/- pairs and patches +/-g make the global additive bias zero. + response, w_des, w_iv, intrinsic, shear = [], [], [], [], [] + for patch_shear in (+G_TRUE, -G_TRUE): + for r, wd, wi in ((0.5, 1.0, 3.0), (1.5, 3.0, 1.0)): + for e in (+0.1, -0.1): + response.append(r) + w_des.append(wd) + w_iv.append(wi) + intrinsic.append(e) + shear.append(patch_shear) + response, w_des, w_iv, intrinsic, shear = map( + np.array, (response, w_des, w_iv, intrinsic, shear) + ) + zeros = np.zeros_like(response) + columns = [ + fits.Column(name="e1_uncal", array=response * shear + intrinsic, format="D"), + fits.Column(name="e2_uncal", array=zeros, format="D"), + fits.Column(name="w_des", array=w_des, format="D"), + fits.Column(name="w_iv", array=w_iv, format="D"), + fits.Column(name="R_g11", array=response, format="D"), + fits.Column(name="R_g12", array=zeros, format="D"), + fits.Column(name="R_g21", array=zeros, format="D"), + fits.Column(name="R_g22", array=response, format="D"), + ] + # The response is valid for this unchanged population and weight identity. + header = fits.Header( + { + "WGHTSCH": weight_type, + "N_ns": len(response), + "SW_ns": np.sum(w_des if weight_type == "des" else w_iv), + } + ) + for key in ("R_S11", "R_S12", "R_S21", "R_S22"): + header[key] = 0.0 + with fits.HDUList( + [fits.PrimaryHDU(header=header), fits.BinTableHDU.from_columns(columns)] + ) as hdul: + hdul.writeto(path) + return {"des": w_des, "iv": w_iv}, shear > 0 + + +@pytest.mark.parametrize("weight_type", ["des", "iv"]) +def test_recalibrated_weighted_shear_uses_weighted_response(tmp_path, weight_type): + """The weighted recalibrated patch shear must recover g=0.02 exactly. + + Patch P has responses 0.5 and 1.5 with opposite w_des/w_iv preferences. + Intrinsic +/- pairs in patches with opposite shears make the global + weighted mean ellipticity zero, so c=0. The patch's weighted uncalibrated + mean is exactly _w*g. Dividing by the matching response (des: 1.25, + iv: 0.75) recovers g; the unweighted response 1.0 gives 1.25*g or 0.75*g. + """ + path = tmp_path / "cat.fits" + weights, in_patch = _write_catalogue(path, weight_type) + w = weights[weight_type] + g1, _ = get_calibrate_e_from_cat(str(path), weight_type=weight_type) + g_hat = np.average(g1[in_patch], weights=w[in_patch]) + assert g_hat == pytest.approx(G_TRUE, rel=0, abs=1e-12), ( + f"weight_type={weight_type!r}: weighted patch shear {g_hat:.6f} " + f"!= truth {G_TRUE}; response and additive bias must share weights" + ) diff --git a/src/sp_validation/tests/test_response_diagnostics.py b/src/sp_validation/tests/test_response_diagnostics.py new file mode 100644 index 000000000..d80276c69 --- /dev/null +++ b/src/sp_validation/tests/test_response_diagnostics.py @@ -0,0 +1,60 @@ +"""Brute-force checks of the explicitly shear-only #23 stage-1 diagnostic.""" + +import numpy as np + +from sp_validation.response_diagnostics import pair_products + + +def test_all_pair_tensor_products_against_brute_force(): + ra = np.array([0.0, 0.03, 0.07, 0.13, 0.21, 0.32]) + dec = np.zeros(6) + w = np.array([1.0, 4.0, 2.0, 3.0, 1.0, 2.0]) + r = np.array( + [ + np.linspace(0.6, 1.1, 6), + np.linspace(-0.1, 0.2, 6), + np.linspace(0.2, -0.05, 6), + np.linspace(0.8, 1.2, 6), + ] + ) + result = pair_products( + ra, dec, w, r.reshape(2, 2, 6), min_sep=1, max_sep=30, nbins=3 + ) + sums = np.zeros((3, 4, 4)) + den = np.zeros(3) + for i in range(6): + for j in range(6): + if i == j: + continue + theta = abs(ra[i] - ra[j]) * 60 + if not 1 <= theta < 30: + continue + k = int(np.log(theta) / (np.log(30) / 3)) + sums[k] += w[i] * w[j] * np.outer(r[:, i], r[:, j]) + den[k] += w[i] * w[j] + np.testing.assert_allclose( + result["products"], sums / den[:, None, None], rtol=2e-7, atol=1e-9 + ) + # Spatial variation is measurable; the global product need not suffice. + assert ( + np.max(abs(result["products"][:, 0, 0] / result["global_products"][0, 0] - 1)) + > 0.02 + ) + + +def test_constant_full_matrix_has_exact_pair_product(): + matrix = np.array([[0.8, 0.1], [-0.02, 0.9]]) + rg = np.repeat(matrix[:, :, None], 6, axis=2) + result = pair_products( + np.arange(6) * 0.05, + np.zeros(6), + np.arange(6) + 1.0, + rg, + min_sep=1, + max_sep=30, + nbins=3, + ) + expected = np.outer(matrix.ravel(), matrix.ravel()) + np.testing.assert_allclose( + result["products"], np.broadcast_to(expected, (3, 4, 4)), rtol=2e-7, atol=1e-9 + ) diff --git a/src/sp_validation/tests/test_response_weights.py b/src/sp_validation/tests/test_response_weights.py new file mode 100644 index 000000000..25ec42567 --- /dev/null +++ b/src/sp_validation/tests/test_response_weights.py @@ -0,0 +1,92 @@ +"""Branch IV and refitted-table regression tests.""" + +import numpy as np +import pytest + +from sp_validation import calibration, response +from sp_validation.tests.test_consistent_response import catalogue + + +def test_extraction_carries_available_branch_errors_but_allows_historical_des(): + data, _, _ = catalogue() + carried = response.branch_error_columns(data) + assert len(carried) == 8 + for key, value in carried.items(): + np.testing.assert_array_equal(value, data[key]) + historical = data[[k for k in data.dtype.names if k not in carried]] + assert response.branch_error_columns(historical) == {} + # Requiring these columns for DES would break existing merged inputs. + calibration.metacal( + historical, + np.ones(len(historical), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + + +def test_iv_branch_variances_and_normalization(): + data, _, _ = catalogue() + data["NGMIX_G1_ERR_1P"] = np.linspace(0.1, 0.8, len(data)) + data["NGMIX_G2_ERR_1M"] = np.linspace(0.8, 0.1, len(data)) + gm = calibration.metacal( + data, np.ones(len(data), bool), weight_scheme="iv", size_corr_ell=False + ) + for b, suffix in (("p1", "1P"), ("m1", "1M")): + expected = 1 / ( + 2 * 0.34**2 + + data[f"NGMIX_G1_ERR_{suffix}"] ** 2 + + data[f"NGMIX_G2_ERR_{suffix}"] ** 2 + ) + np.testing.assert_allclose(gm.branch_weights[b], expected) + expected = ( + np.average(gm.branches["ns"], weights=gm.branch_weights["p1"], axis=1) + - np.average(gm.branches["ns"], weights=gm.branch_weights["m1"], axis=1) + ) / 0.02 + np.testing.assert_allclose(gm.R_selection[:, 0], expected) + assert abs(expected[0]) > 0.01 + + +def test_refit_w_jackknife_matches_retained_training_sample(): + data, _, _ = catalogue() + # Unequal group noise at the same SNR changes fitted W on deletion. + for b in ("NOSHEAR", "1P", "1M", "2P", "2M"): + data[f"NGMIX_G1_{b}"][:20] += 0.3 + gm = calibration.metacal( + data, + np.ones(len(data), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + groups = np.repeat(np.arange(4), 20) + labels, jk, cov = gm.response_jackknife(groups, refit_w=True) + for j, label in enumerate(labels): + keep = groups != label + retained = data[keep] + other = calibration.metacal( + retained, + np.ones(len(retained), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + ) + np.testing.assert_allclose(jk["c_g"][j], other.c_g, atol=1e-13) + np.testing.assert_allclose(jk["total"][j], other.R, atol=1e-13) + fixed = gm.response_jackknife(groups)[1] + assert np.max(abs(jk["c_g"] - fixed["c_g"])) > 1e-3 + vectors = np.concatenate([x.reshape(len(labels), -1) for x in jk.values()], axis=1) + delta = vectors - vectors.mean(axis=0) + np.testing.assert_allclose(cov, 0.75 * delta.T @ delta) + weights = gm.refit_weights(np.ones(len(data), bool)) + for b in response.BRANCHES: + np.testing.assert_allclose(weights[b], gm.branch_weights[b]) + + +def test_iv_missing_branch_errors_requires_reextraction(): + data, _, _ = catalogue() + data = data[[k for k in data.dtype.names if k != "NGMIX_G1_ERR_1P"]] + with pytest.raises(ValueError, match="regenerate.*comprehensive"): + calibration.metacal( + data, np.ones(len(data), bool), weight_scheme="iv", size_corr_ell=False + ) diff --git a/src/sp_validation/tests/test_sparse_des.py b/src/sp_validation/tests/test_sparse_des.py new file mode 100644 index 000000000..c55f8e119 --- /dev/null +++ b/src/sp_validation/tests/test_sparse_des.py @@ -0,0 +1,126 @@ +"""Minimum-count DES moments and identical full/deleted training rules.""" + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import calibration, response +from sp_validation.tests.test_consistent_response import catalogue + + +def sparse_fixture(): + # Two nearly round objects would otherwise get W=640000. + snr = np.r_[np.repeat(20.0, 40), [100.0, 100.0]] + size = np.ones(len(snr)) + e = np.full((2, len(snr)), 0.2) + e[:, -2:] = 0.001 + r = np.full(len(snr), 0.8) + edges = np.array([10.0, 50.0, 500.0]), np.array([0.5, 2.0, 3.0]) + return snr, size, e, r, *edges + + +def test_sparse_cell_pools_moments_not_weights(): + args = sparse_fixture() + raw = response.fit_w(*args) + pooled = response.fit_w(*args, min_count=3) + assert raw[2][1, 0] > 600000 + expected = 0.8**2 / np.mean(args[2] ** 2) + np.testing.assert_allclose(pooled[2][1, 0], expected) + assert pooled[2][1, 0] < 17 + # Dense cell is bitwise unchanged; training counts stay unmerged. + assert pooled[2][0, 0] == raw[2][0, 0] + np.testing.assert_array_equal(pooled[3], raw[3]) + # A newly occupied artificial-branch cell uses the same pooled table. + np.testing.assert_allclose(response.evaluate_w(pooled, [100.0], [2.5]), expected) + + +def test_sparse_deletion_is_deterministic_and_order_independent(): + snr, size, e, r, es, er = sparse_fixture() + for deleted in (0, 40, 41): + keep = np.arange(len(snr)) != deleted + args = (snr[keep], size[keep], e[:, keep], r[keep], es, er) + first = response.fit_w(*args, min_count=30) + second = response.fit_w(*args, min_count=30) + reverse = response.fit_w( + snr[keep][::-1], + size[keep][::-1], + e[:, keep][:, ::-1], + r[keep][::-1], + es, + er, + min_count=30, + ) + np.testing.assert_array_equal(first[2], second[2]) + np.testing.assert_allclose(first[2], reverse[2], rtol=1e-14) + assert first[2][1, 0] < 17 + + +def test_fully_populated_table_is_bitwise_unchanged(): + snr = np.repeat([20.0, 20.0, 100.0, 100.0], 30) + size = np.repeat([1.0, 2.5, 1.0, 2.5], 30) + e = np.array([np.linspace(0.1, 0.4, len(snr))] * 2) + args = ( + snr, + size, + e, + np.full(len(snr), 0.8), + np.array([10.0, 50.0, 500.0]), + np.array([0.5, 2.0, 3.0]), + ) + for actual, expected in zip( + response.fit_w(*args, min_count=30), response.fit_w(*args), strict=True + ): + np.testing.assert_array_equal(actual, expected) + + +def test_neighbourhood_grows_until_threshold_and_handles_small_total(): + snr = np.r_[15.0, np.repeat(150.0, 10), np.repeat(1500.0, 40)] + size = np.ones(len(snr)) + e = np.array([np.r_[0.001, np.repeat(0.1, 10), np.repeat(0.3, 40)]] * 2) + args = ( + snr, + size, + e, + np.ones(len(snr)), + np.array([10.0, 100.0, 1000.0, 10000.0]), + np.array([0.5, 2.0]), + ) + near = response.fit_w(*args, min_count=10) + far = response.fit_w(*args, min_count=30) + np.testing.assert_allclose(near[2][0, 0], 1 / np.mean(e[:, :11] ** 2)) + np.testing.assert_allclose(far[2][0, 0], 1 / np.mean(e**2)) + small = response.fit_w(*args, min_count=100) + np.testing.assert_allclose(small[2], 1 / np.mean(e**2)) + with pytest.raises(ValueError, match="nonnegative"): + response.fit_w(*args, min_count=-1) + + +def test_production_refit_uses_same_minimum_count_rule(tmp_path): + d, _, _ = catalogue() + gm = calibration.metacal( + d, + np.ones(len(d), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + des_min_count=30, + ) + keep = np.arange(len(d)) % 4 != 0 + refitted = gm.refit_weights(keep) + rebuilt = calibration.metacal( + d[keep], + np.ones(keep.sum(), bool), + weight_scheme="des", + size_corr_ell=False, + des_num_bins=2, + des_min_count=30, + ) + for b in response.BRANCHES: + np.testing.assert_allclose(refitted[b][keep], rebuilt.branch_weights[b]) + path = tmp_path / "response.fits" + fits.HDUList([fits.PrimaryHDU()]).writeto(path) + calibration.save_response_product(path, gm) + with fits.open(path) as hdus: + assert hdus[0].header["DESNMIN"] == 30 + assert "pooled moments" in hdus[0].header["DESMERGE"] + np.testing.assert_array_equal(hdus["W_COUNT"].data, gm.W[3]) diff --git a/workflow/bin/spv b/workflow/bin/spv new file mode 100755 index 000000000..6e7226657 --- /dev/null +++ b/workflow/bin/spv @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# spv -- launcher for sp_validation's image-sims workflow; the counterpart of +# ShapePipe's workflow/bin/sp. +# +# Usage: +# spv run -c FILE [-t TARGET]... [SNAKEMAKE ARGS...] +# run the image-sims workflow for the run config FILE. +# Default target: im_mbias (init -> extract -> calibrate +# -> m-bias). Snapshots the code into the state dir first +# and runs out of the copy, so editing the checkout +# mid-run cannot reach the jobs. -n is a dry run. +# spv status -c FILE [FLAGS...] +# spv status DIR [FLAGS...] +# disk-scan status of ShapePipe campaigns: one sp run +# config, or every sp_*.yaml in DIR (default flag -S). +# Read-only; see workflow/scripts/status_sims.py --help. +# spv -c FILE -- [SNAKEMAKE ARGS...] +# snakemake on this run's existing snapshot (--unlock, +# --summary, ...); no new snapshot. +# +# Configs: workflow/image_sims/config.yaml is the main config (defaults); FILE +# is the user's run config, which must set image_sims.grids_base and +# image_sims.shapepipe_repo and may override anything else. Snakemake +# deep-merges the two. image_sims.sp_validation_repo is set by spv (to the +# snapshot) and need not be in FILE. +# +# Environment: +# SPV_PROFILE workflow/profiles/ (default candide) +# SPV_STATE_DIR snakemake state + code snapshot +# (default /spv-state) +# TMPDIR, APPTAINER_TMPDIR +# temporary directories of the head, the jobs and +# apptainer; on candide REQUIRED, absolute, and not under +# /tmp or /scratch (see tmp_guard) +# +# Example (candide): +# export TMPDIR=/n23data1/$USER/tmp APPTAINER_TMPDIR=/n23data1/$USER/tmp +# workflow/bin/spv run -c ~/runs/grid_3/spv_grid_3.yaml -n +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROFILE="${SPV_PROFILE:-candide}" + +usage() { sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d; s/^# \{0,1\}//'; } + +# --- temporary directories --------------------------------------------------- +# The slurm executor exports this shell's environment to every job, TMPDIR +# literally. On candide the login node's TMPDIR (/scratch/$USER/tmp) cannot be +# created on a compute node, so Slurm falls back to the node's shared 31 GB +# /tmp; inside the container the profile's --cleanenv drops TMPDIR and the host +# /tmp is mounted. Where apptainer cannot mount the image it unpacks all of it +# into its temporary directory -- that filled /tmp on n35 (2026-09-30). Hence: +# on candide both directories must be a path every node sees, outside /tmp and +# /scratch, and the same path goes into the container (snapshot_code). +tmp_guard() { + local v val bad + for v in TMPDIR APPTAINER_TMPDIR; do + val="${!v:-}"; bad="" + if [ -z "$val" ]; then bad="unset" + elif [[ "$val" != /* ]]; then bad="not absolute" + elif [ "$val" = /tmp ] || [[ "$val" == /tmp/* ]]; then bad="under /tmp" + elif [ "$PROFILE" = candide ] && [[ "$val" == /scratch/* ]]; then + bad="under /scratch, which compute nodes cannot create" + fi + if [ -n "$bad" ]; then + if [ "$PROFILE" = candide ]; then + echo "spv: $v is $bad (${val:-}). On candide set TMPDIR and" \ + "APPTAINER_TMPDIR to a directory every node sees, e.g." \ + "/n23data1/\$USER/tmp, before \`spv run\`." >&2 + exit 2 + fi + echo "spv: warning: $v is $bad (${val:-})" >&2 + continue + fi + mkdir -p "$val" + done + [ -z "${APPTAINER_TMPDIR:-}" ] || export SINGULARITY_TMPDIR="$APPTAINER_TMPDIR" +} + +# --- arguments --------------------------------------------------------------- +cmd="${1:-}" +case "$cmd" in ""|-h|--help) usage; exit 0 ;; esac +[ "$cmd" = run ] || [ "$cmd" = status ] && shift || cmd=passthrough + +RUN_CONFIG=""; TARGETS=(); REST=() +while [ $# -gt 0 ]; do + case "$1" in + -c|--config-file) [ $# -ge 2 ] || { echo "spv: $1 needs a file" >&2; exit 2; } + RUN_CONFIG="$2"; shift 2 ;; + -t|--target) [ $# -ge 2 ] || { echo "spv: $1 needs a target" >&2; exit 2; } + TARGETS+=("$2"); shift 2 ;; + --) shift; REST+=("$@"); break ;; + *) REST+=("$1"); shift ;; + esac +done + +# --- status: read-only, no state dir ------------------------------------------- +if [ "$cmd" = status ]; then + STATUS="$REPO/workflow/scripts/status_sims.py" + flags=(); dir="" + for a in ${REST[@]+"${REST[@]}"}; do + if [ -z "$dir" ] && [ -d "$a" ]; then dir="$a"; else flags+=("$a"); fi + done + [ ${#flags[@]} -gt 0 ] || flags=(-S) + if [ -n "$RUN_CONFIG" ]; then + exec python3 "$STATUS" -c "$RUN_CONFIG" "${flags[@]}" + fi + [ -n "$dir" ] || { echo "spv status: give -c or a directory" >&2; exit 2; } + shopt -s nullglob; cfgs=("$dir"/sp_*.yaml) + [ ${#cfgs[@]} -gt 0 ] || { echo "spv status: no sp_*.yaml in $dir" >&2; exit 2; } + rc=0 + for c in "${cfgs[@]}"; do + echo; echo "############## $(basename "$c" .yaml) ##############" + python3 "$STATUS" -c "$c" "${flags[@]}" || rc=$? + done + exit $rc +fi + +# --- run / passthrough: the campaign's state dir -------------------------------- +[ -n "$RUN_CONFIG" ] || { echo "spv: -c is required" >&2; exit 2; } +[ -f "$RUN_CONFIG" ] || { echo "spv: run config $RUN_CONFIG does not exist" >&2; exit 2; } +RUN_CONFIG="$(cd "$(dirname "$RUN_CONFIG")" && pwd)/$(basename "$RUN_CONFIG")" +[ -d "$REPO/workflow/profiles/$PROFILE" ] || { + echo "spv: no profile workflow/profiles/$PROFILE (SPV_PROFILE=$PROFILE)" >&2; exit 2; } +command -v snakemake >/dev/null || { echo "spv: snakemake is not on PATH" >&2; exit 2; } + +GRIDS_BASE="$(python3 - "$REPO/workflow/image_sims/config.yaml" "$RUN_CONFIG" <<'PY' +import sys, yaml +main, user = (yaml.safe_load(open(p)) or {} for p in sys.argv[1:3]) +val = (user.get("image_sims") or {}).get("grids_base") \ + or (main.get("image_sims") or {}).get("grids_base") +print(val or "") +PY +)" +[ -n "$GRIDS_BASE" ] || { echo "spv: image_sims.grids_base is not set in $RUN_CONFIG" >&2; exit 2; } +STATE_DIR="${SPV_STATE_DIR:-$GRIDS_BASE/spv-state}" +SNAPSHOT="$STATE_DIR/code" +tmp_guard + +# --- the launch code snapshot --------------------------------------------------- +# As in ShapePipe's sp: the jobs re-read the Snakefile, rules and scripts when +# they start, so a run executed straight from the checkout picks up whatever is +# edited there meanwhile. spv copies what it launches into $STATE_DIR/code and +# runs entirely out of it; image_sims.sp_validation_repo points there. Refreshed +# wholesale on every `spv run`. +snapshot_code() { + mkdir -p "$SNAPSHOT" + rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ + "$REPO/workflow" "$REPO/src" "$REPO/scripts" "$REPO/config" "$SNAPSHOT/" + cp "$RUN_CONFIG" "$SNAPSHOT/run_config.yaml" + + # Temporary directory inside the container: --cleanenv drops TMPDIR, so the + # head's TMPDIR goes in as a literal --env (tempfile, joblib). It must lie + # under a --bind, or the container could not write it. + python3 - "$SNAPSHOT/workflow/profiles/$PROFILE/config.yaml" "${TMPDIR:-}" <<'PY' +import re, sys, yaml +path, tmp = sys.argv[1], sys.argv[2] +prof = yaml.safe_load(open(path)) +args = prof.get("apptainer-args", "") +if tmp and not tmp.startswith("/tmp"): + roots = [r for b in re.findall(r"--bind (\S+)", args) + for r in (s.split(":")[0] for s in b.split(","))] + if not any(tmp == r or tmp.startswith(r.rstrip("/") + "/") for r in roots): + sys.exit(f"spv: TMPDIR {tmp} is not under any --bind of the profile " + f"({', '.join(roots)}); the container could not write it") + args = re.sub(r"\s*--env (TMPDIR|JOBLIB_TEMP_FOLDER)=\S+", "", args) + prof["apptainer-args"] = f"{args} --env TMPDIR={tmp} --env JOBLIB_TEMP_FOLDER={tmp}" + with open(path, "w") as fh: + fh.write("# Launch snapshot of the profile, rewritten by spv.\n") + yaml.safe_dump(prof, fh, sort_keys=False) + print(f"spv: temporary directory {tmp} (head, jobs and container)", file=sys.stderr) +PY + + python3 - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' +import json, subprocess, sys, time +out, repo = sys.argv[1], sys.argv[2] +def git(*a): + p = subprocess.run(["git", "-C", repo, *a], capture_output=True, text=True) + return p.stdout.strip() if p.returncode == 0 else None +status = git("status", "--porcelain") +json.dump({"_comment": "Code snapshot taken by `spv run`; the run executes " + "this tree, never the checkout.", + "source": repo, "taken_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "head": git("rev-parse", "HEAD"), + "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(status), + "dirty_files": status.splitlines() if status else []}, + open(out, "w"), indent=2) +PY + echo "spv: code snapshot refreshed at $SNAPSHOT" >&2 +} + +sm() { + [ -f "$SNAPSHOT/workflow/image_sims/Snakefile" ] || { + echo "spv: no snapshot at $SNAPSHOT yet; run \`spv run\` first" >&2; exit 2; } + cd "$STATE_DIR" + # Targets and the caller's arguments BEFORE --configfile/--config: both take + # one-or-more values and would swallow a target that follows them. + snakemake ${TARGETS[@]+"${TARGETS[@]}"} ${REST[@]+"${REST[@]}"} \ + -s "$SNAPSHOT/workflow/image_sims/Snakefile" \ + --profile "$SNAPSHOT/workflow/profiles/$PROFILE" \ + --directory "$STATE_DIR" \ + --configfile "$SNAPSHOT/run_config.yaml" \ + --config "image_sims={\"sp_validation_repo\": \"$SNAPSHOT\"}" +} + +if [ "$cmd" = run ]; then + snapshot_code + [ ${#TARGETS[@]} -gt 0 ] || TARGETS=(im_mbias) +fi +sm diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile index 7420ae686..0a95da645 100644 --- a/workflow/image_sims/Snakefile +++ b/workflow/image_sims/Snakefile @@ -4,20 +4,15 @@ Run the sp_validation-side chain (merge -> extract -> calibrate -> m-bias), optionally including the ShapePipe pipeline stage, without pulling in the cosmology-validation config the top-level ``workflow/Snakefile`` requires. -The one drive command on candide, with the committed SLURM profile owning -scheduling and container wrapping: +Launch it with ``workflow/bin/spv run -c my_run.yaml`` (``-n`` for a dry run), +which snapshots the code, sets the temporary directories and passes the profile; +see ``spv --help``. ``configfile:`` below loads the operational defaults (the +main config), so ``my_run.yaml`` need only carry the science knobs and any +override; Snakemake deep-merges the two. - snakemake --profile workflow/profiles/candide \\ - -s workflow/image_sims/Snakefile \\ - im_mbias --configfile my_run.yaml - -``configfile:`` below loads the operational defaults, so ``my_run.yaml`` need -only carry the science knobs (and any operational override); Snakemake -deep-merges the two. Off-cluster, drop ``--profile`` and add ``-j N``. - -Put targets *before* ``--configfile``: it takes one-or-more paths, so a target -after it is swallowed as a config path ("No such file: im_mbias"). Always -dry-run first with ``-n``. +Driving snakemake directly also works: put targets *before* ``--configfile`` +(it takes one-or-more paths, so a target after it is swallowed as a config +path), and on candide set TMPDIR/APPTAINER_TMPDIR yourself. The same rules are also included in the main workflow under ``if "image_sims" in config``. @@ -34,7 +29,9 @@ sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), ".."))) import common -configfile: "workflow/image_sims/config.yaml" +# Next to this file, not relative to the working directory: spv runs snakemake +# from the campaign's state directory, out of a code snapshot. +configfile: os.path.join(str(workflow.basedir), "config.yaml") include: "../rules/image_sims.smk" diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index f5f3e3c0e..a2622945f 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -1,37 +1,29 @@ # Image-simulation m-bias workflow configuration. # # OPERATIONAL keys default here and nowhere else -- the .smk reads them bare, so -# this file is their single home. SCIENCE keys have no default anywhere and -# appear below only as commented template lines: they fix the estimator's -# behaviour, so each run must state them in a run config layered on top. A run -# config that omits one fails at DAG parse, naming the key; an unknown key under +# this file is their single home. Run-specific SCIENCE keys appear below as +# commented template lines and must be stated in a layered run config. +# DES has one shared production policy: use an independent named data table; +# sample-fitted tables require the explicit diagnostic override below. +# A run missing a required science key fails at DAG parse; an unknown key under # `image_sims:` fails as a typo. image_sims: - # --- container -------------------------------------------------------- - # One image for the whole chain: the sp_validation image is built FROM the - # ShapePipe image, so it carries both stacks. CI builds it from the uv lock -- - # an unlocked build drifts NumPy past numba's ceiling and the ngmix stage dies - # ("Numba needs NumPy 2.4 or less"). - # - # `null` means the workflow's one image: your own .sif if you have pulled one - # with `spv-container pull`, else the registry tag for Snakemake to autopull - # (workflow/README.md). Neither is repeated here. - # Override with a local .sif path, or with a branch tag - # (docker://ghcr.io/cosmostat/sp_validation:) to run a - # branch's own CI image. - sif: null - - # --- repositories ----------------------------------------------------- + # --- repositories (REQUIRED in the run config; no default) ------------- # Bound into the image; both repos' src go on PYTHONPATH so this branch's - # code wins over the baked copies. - shapepipe_repo: /n17data/cdaley/unions/code/shapepipe - sp_validation_repo: /n17data/cdaley/unions/code/sp_validation + # code wins over the baked copies. Per-user checkouts, so there is nothing + # sensible to default to -- a default here would silently run someone else's + # code. A run config that omits one fails at parse, naming it. + # + # shapepipe_repo: /path/to/shapepipe + # sp_validation_repo: /path/to/sp_validation # --- data and run directories ---------------------------------------- - # grids_base is the run/output root: one sub-directory per simulation. - grids_base: /n17data/cdaley/unions/scratch-wf/imsims-run/grids + # grids_base is the run/output root (one sub-directory per simulation), and + # is REQUIRED in the run config -- it is where this user's campaign writes. + # + # grids_base: /path/to/grids input_sims_base: /n09data/hervas/skills_out psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle @@ -42,21 +34,63 @@ image_sims: # sheared branches. Injected shear is NOT set here -- it is parsed from each # branch's basic_info.txt by im_manifest. [operational] branches: ["1z2z", "1p2z", "1m2z", "1z2p", "1z2m"] - # Tiles to process: an explicit list, the one tile-input mechanism. + # Tiles to process: an explicit list, the one tile-input mechanism. Leave + # out tiles without data: a tile with no detections fails its branch's + # ShapePipe campaign, and so im_shapepipe. In SKiLLS the coadds of grid_1 + # 248.288, grid_2 259.285, grid_3 265.285 and grid_4 281.281/282.281 are + # all zeros, and 1m2z_2's 257.289 is truncated. tile_ids: ["233.293", "237.292", "238.292"] # --- calibration ------------------------------------------------------ shape: ngmix # [operational] - # ShapePipe cfis configs (final_cat.param etc.); default is - # {shapepipe_repo}/example/cfis_image_sims once #766 lands. [operational] - config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims - psf_model: psfex # [operational] - n_smp: -1 # [operational] - # Extract/calibrate scripts run from the sp_validation repo checkout (branch - # code, not the baked copies). Point elsewhere for a different checkout. + + # --- ShapePipe processing (ShapePipe's own Snakemake workflow) --------- + # im_shapepipe runs `{shapepipe_repo}/workflow/bin/sp run` once per branch + # with input_type: image_sims, so shapepipe_repo must be a checkout carrying + # that workflow mode. The config chain is the checkout's + # workflow/config/cfis_image_sims -- change processing there, not here. + # PSF model, as ShapePipe's workflow spells it: `fake` (the true simulation + # PSF; the star-free *_grid_N sims) or `psfex` / `mccd` (the sims with + # stars: non-grid *_N, tiles from N = 1..4 with exp_num: 5; *_star_grid has + # inputs for 1z2z only and gives no m). [operational] + psf_model: fake + # ShapePipe profile (shapepipe_repo/profiles/) that submits the + # campaign's SLURM jobs. [operational] + sp_profile: candide + # Concurrent SLURM jobs per branch campaign (`sp run --jobs`); the profile's + # `sp_campaign` resource caps how many branch campaigns run at once, so the + # cluster sees up to that many times this. [operational] + sp_jobs: 50 + # Rolling exposure-store reclamation (ShapePipe `clean:`): an exposure's + # store (~8 GB) is deleted once every tile reading it has its vignets. # [operational] - extract_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/extract_info.py - calibrate_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/calibrate_comprehensive_cat.py + clean_exposures: true + # Where each tile's ShapePipe vignette store lives (ShapePipe run-config + # `tile_store_root`, bound to /local/scratch by `sp run`). null keeps the + # profile's node-local store -- on candide the node's 31 GB /tmp, which + # dense star-sim tiles (~5 GB each) and failed groups' orphans fill, failing + # every later tile group on that node. Set a shared directory for the star + # sims (all branches and runs can share one root). [operational] + tile_store_root: null + # Extract/calibrate scripts default to the sp_validation_repo checkout above + # (as compute_m_bias_image_sims.py already did), so branch code runs rather + # than the copies baked into the image. Set one only to point at a DIFFERENT + # checkout. + # + # extract_script: /other/checkout/scripts/calibration/extract_info.py + # calibrate_script: /other/checkout/scripts/calibration/calibrate_comprehensive_cat.py + + # DES sims must apply the same fixed table as the data they calibrate. + # Supply a named modern calibration FITS product (W_DES/W_EDGES/W_COUNT, + # with coordinate metadata), not a released catalogue with only w_des. + # No public v1.4.6.3 table with verified compatible edges is available. + # This intentionally blocks a DES production run until its data product is set. + des_weight_source: external + des_weight_product: null + # des_weight_product: + # name: + # path: /absolute/path/to/shape_catalog_cut_ngmix.fits + # des_weight_source: self # explicitly sample-fitted diagnostic, not production # --- science knobs (REQUIRED in the run config; no default) ----------- # Copy these into your run config and set them. There is deliberately no @@ -66,11 +100,8 @@ image_sims: # # mask_config: config/calibration/mask_v1.X.9_im_sim.yaml # relative to sp_validation_repo # match_radius_deg: 0.0002 # RA/Dec pair-match radius, degrees - # w_cols: [none, w_iv] # weight schemes to compute in one run; "none" - # # (or null) = unit weights (#227); first entry is - # # the primary/headline result -- lead with "none" - # # for the fiducial unweighted estimator. The - # # deprecated scalar `w_col` is still accepted. - # pair_match: true # match +g/-g object-by-object (per-object cancellation) + # w_cols: [w_des] # must match the calibration's weight_scheme: des + # pair_match: false # full populations, branch-specific normalization; + # # modern products refuse symmetric pair weighting # n_bootstrap: 500 # bootstrap resamples for the errors # bootstrap_seed: 42 # seed for the bootstrap RNG (makes errors reproducible) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 39649717c..009dccdf1 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -8,6 +8,15 @@ # full story. executor: slurm +# Concurrent SLURM jobs. Required: snakemake refuses remote execution without +# a job cap ("maximum number of parallel jobs ... has to be specified"). +jobs: 50 +# Concurrent ShapePipe campaigns (im_shapepipe, one per shear branch; each +# submits its own jobs through ShapePipe's profile). A disk bound -- see the +# rule. Raise it on the command line (--resources sp_campaign=5) when the run +# dir has room for all five branches' exposure stores at once. +resources: + sp_campaign: 2 # --- mirrored in workflow/profiles/default/config.yaml ---------------------- software-deployment-method: apptainer @@ -48,7 +57,7 @@ default-resources: slurm_partition: "comp,pscomp" runtime: "60m" cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n36'" + slurm_extra: "'--exclude=n17,n34,n35,n36'" # Snakemake refuses a real run on a remote executor without a job bound. jobs: 100 diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index cc9923912..210d20d57 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -16,7 +16,8 @@ check at load rejects an unknown key (typo) and a missing science key (see The five simulations per grid are the reference ``1z2z`` (no input shear) plus the ``+/-`` shear pairs ``1p2z``/``1m2z`` (g1) and ``1z2p``/``1z2m`` (g2); the -m-bias estimator matches each to the reference by RA/Dec. +modern m-bias estimator uses full populations with separately normalized +branch weights (pair_match: false), matching the stored response. """ import os @@ -49,15 +50,35 @@ _OPERATIONAL_KEYS = { "sims_type", "branches", "shape", - "config_dir", "psf_model", - "n_smp", + "sp_profile", + "sp_jobs", + "clean_exposures", + "tile_store_root", +} +# Optional keys: recognized but not required; defaults applied in code below. +_OPTIONAL_KEYS = { + # Overrides only: each has a working default derived below from + # sp_validation_repo, so pointing at a different checkout is opt-in and a + # missing key cannot silently select someone else's code. "extract_script", "calibrate_script", + # DES weight table policy; defaults in config.yaml. + "des_weight_source", + "des_weight_product", + # The container image. Optional because it has a working default chain + # (sp_validation.container.resolve_image: local sandbox, then local sif, + # then the registry tag), unlike the structural keys below, which have no + # sensible default at all. On candide it comes from the profile, whose + # `config:` sets the cluster's image -- a machine-specific path belongs + # with the machine, not in this shared file. + "sif", + # Exposures from another realization than the tiles (non-grid sims only; + # default: num). See EXP_NUM below and config.yaml. + "exp_num", } # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { - "sif", "shapepipe_repo", "sp_validation_repo", "grids_base", @@ -67,7 +88,11 @@ _STRUCTURAL_KEYS = { "tile_ids", } _ALLOWED_KEYS = ( - _SCIENCE_KEYS | _DEPRECATED_KEYS | _OPERATIONAL_KEYS | _STRUCTURAL_KEYS + _SCIENCE_KEYS + | _DEPRECATED_KEYS + | _OPERATIONAL_KEYS + | _OPTIONAL_KEYS + | _STRUCTURAL_KEYS ) _unknown = set(IMSIM) - _ALLOWED_KEYS @@ -91,13 +116,25 @@ if _missing_structural: f"{_missing_structural} -- set them in the run config" ) +if "w_des" in IMSIM["w_cols"]: + if IMSIM["des_weight_source"] not in ("external", "self"): + raise ValueError("image_sims: des_weight_source must be external or self") + _product = IMSIM["des_weight_product"] + if IMSIM["des_weight_source"] == "external" and ( + not isinstance(_product, dict) or not _product.get("name") or not _product.get("path") + ): + raise ValueError( + "DES image sims require des_weight_product with name and path; " + "des_weight_source: self is diagnostic only" + ) + # --- container ------------------------------------------------------------ # Every compute rule carries ``container: SIF`` rather than inheriting a # module-level default: these rules are also included from the top-level # workflow/Snakefile, whose module default is the cosmology image (no ShapePipe # stack). Binds come from the driving profile's ``apptainer-args``. A null # ``sif`` resolves to the workflow's one image (see workflow/image_sims/config.yaml). -SIF = common.resolve_container(IMSIM["sif"]) +SIF = common.resolve_container(IMSIM.get("sif")) # --- repositories (bound into the image; branch code overrides) ----------- SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] @@ -118,6 +155,15 @@ SIMS_TYPE = IMSIM["sims_type"] _SUFFIX = f"_{SIMS_TYPE}_{NUM}" if SIMS_TYPE == "grid" else f"_{NUM}" SIM_BASES = list(IMSIM["branches"]) SIMS = [f"{base}{_SUFFIX}" for base in SIM_BASES] +# Optional ``exp_num`` pulls exposures from a different realization than tiles: +# tiles stay on ``{branch}_{num}``, exposures come from ``{branch}_{exp_num}``. +# Non-grid sims only (grid names embed sims_type); see config.yaml for the +# blended use case. +EXP_NUM = IMSIM.get("exp_num", NUM) +if SIMS_TYPE == "grid": + SIM_EXP_BASE = {sim: sim for sim in SIMS} +else: + SIM_EXP_BASE = {f"{base}{_SUFFIX}": f"{base}_{EXP_NUM}" for base in SIM_BASES} MANIFEST = f"{GRIDS_BASE}/manifest.yaml" BUILD_MANIFEST = f"{SPV_REPO}/workflow/scripts/im_build_manifest.py" @@ -129,18 +175,36 @@ TILE_IDS = list(IMSIM["tile_ids"]) SHAPE = IMSIM["shape"] MASK_CONFIG = IMSIM["mask_config"] # e.g. config/calibration/mask_v1.X.9_im_sim.yaml PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py" -# ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param). -CONFIG_DIR = IMSIM["config_dir"] + +# --- ShapePipe processing: ShapePipe's own workflow ------------------------ +# ``sp`` is ShapePipe's workflow launcher; SP_CONFIG_DIR is the config chain it +# selects for ``input_type: image_sims``. The merge and extract steps read that +# dir's ``final_cat.param`` (through the per-sim ``cfis`` link), so the column +# list and the processing that wrote the columns come from one checkout. +SP_LAUNCHER = f"{SHAPEPIPE_REPO}/workflow/bin/sp" +SP_CONFIG_DIR = f"{SHAPEPIPE_REPO}/workflow/config/cfis_image_sims" +PSF_MODEL = IMSIM["psf_model"] +SP_PROFILE = IMSIM["sp_profile"] +SP_JOBS = IMSIM["sp_jobs"] +CLEAN_EXPOSURES = bool(IMSIM["clean_exposures"]) +# Where each tile_shape group's vignette store lives (ShapePipe's +# ``tile_store_root``; ``sp run`` binds it to /local/scratch). None keeps the +# profile's node-local bind -- and keeps the key out of the run config, so +# campaigns launched before it existed see the same params. +TILE_STORE_ROOT = IMSIM["tile_store_root"] # ShapePipe scripts live in the ShapePipe repo (also baked into its image). -CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" -RUN_JOB = f"{SHAPEPIPE_REPO}/scripts/sh/run_job_sp_canfar_v2.0.bash" # Extract/calibrate run from the sp_validation *repo* checkout (bind-mounted), # not the baked copies: the container tracks the branch but lags it, and the # image-sims path needs branch-only fixes (star-catalogue-optional extract, # FITS-aware CalibrateCat.read_cat). Overridable for a different checkout. -EXTRACT_INFO = IMSIM["extract_script"] -CALIBRATE = IMSIM["calibrate_script"] +EXTRACT_INFO = IMSIM.get( + "extract_script", f"{SPV_REPO}/scripts/calibration/extract_info.py" +) +CALIBRATE = IMSIM.get( + "calibrate_script", + f"{SPV_REPO}/scripts/calibration/calibrate_comprehensive_cat.py", +) # m-bias is *this branch's* extracted core, injected on PYTHONPATH. COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" @@ -151,8 +215,8 @@ COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" # * PYTHONPATH prepends both repos' ``src`` so Python resolves the worktree # build ahead of the copies baked into the image -- the branch's code runs # without an image rebuild. Packages only: the bash and python entry points -# are invoked at the repo paths from config (RUN_JOB, CREATE_FINAL_CAT, -# EXTRACT_INFO, ...), not shadowed by PYTHONPATH. +# are invoked at the repo paths from config (EXTRACT_INFO, +# CALIBRATE, ...), not shadowed by PYTHONPATH. # * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded # via getexpanded) at this run's PSF dictionary. # * OMP_NUM_THREADS=1 rides here rather than in the SLURM profile: the chain @@ -166,7 +230,45 @@ _ENV_PREFIX = ( f"PSF_DICT={PSF_DICT} OMP_NUM_THREADS=1 " ) -JOB_MASK = sum([1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]) + +def sp_run_config(sim): + """ShapePipe workflow run config for one shear branch (one ``sp`` campaign). + + It REPLACES ShapePipe's committed ``workflow/config.yaml`` (``bin/sp``'s + ``SP_RUN_CONFIG``), so every key the campaign needs is stated; processing + knobs this config does not state (``ngmix_chunks``) take the ShapePipe + Snakefile's defaults, never a copy here. ``clean_tiles`` stays off: the + merge reads each tile's make_cat output from the tile store. ``clean`` + (rolling exposure-store reclamation) is ``clean_exposures`` -- a 40-tile + branch touches ~300 exposures at ~8 GB of stores each. + """ + run_dir = f"{GRIDS_BASE}/{sim}" + cfg = { + "input_type": "image_sims", + "psf_model": PSF_MODEL, + "tile_list": f"{run_dir}/tiles_{sim}.txt", + "inputs": { + "tiles": f"{INPUT_SIMS_BASE}/{sim}/images/SP_tiles", + "exposures": f"{INPUT_SIMS_BASE}/{SIM_EXP_BASE[sim]}/images/SP_exp", + }, + "container": SIF_PIPELINE, + "outputs": { + "run_dir": run_dir, + "index_db": f"{run_dir}/index/run_index.sqlite", + }, + "clean": CLEAN_EXPOSURES, + "clean_tiles": False, + "clean_ignore_tiles": [], + } + if PSF_MODEL == "fake": + cfg["psf_dict"] = PSF_DICT + if TILE_STORE_ROOT: + cfg["tile_store_root"] = TILE_STORE_ROOT + return cfg + + +localrules: + im_pipeline, wildcard_constraints: @@ -187,18 +289,9 @@ rule im_init_all: expand(f"{GRIDS_BASE}/{{sim}}/params.py", sim=SIMS), -rule im_pipeline_all: +rule im_shapepipe_all: input: - expand( - f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done", - sim=SIMS, - tile=TILE_IDS, - ), - - -rule im_merge_all: - input: - expand(f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", sim=SIMS), + expand(f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", sim=SIMS), rule im_extract_all: @@ -252,18 +345,18 @@ rule im_manifest: rule im_init: - """Stage per-sim run directory: params.py, mask config, ShapePipe configs, - and the raw SKiLLS image inputs. + """Stage per-sim run directory: params.py, mask config, tile list, and the + ``cfis`` link to ShapePipe's image-sims config dir. ``params_im_sim.py`` derives the field name from the directory basename, so - the same template serves every sim; ``config_mask.yaml`` and ``cfis`` are - symlinks the downstream calibration and merge steps read from cwd. - - ``input_tiles``/``input_exp`` are top-level symlinks to the raw SKiLLS tile - and exposure images; ShapePipe's ``get_images_runner`` resolves them via - ``$SP_DIR/input_{tiles,exp}`` (``$SP_DIR`` is the run dir). ``run_job`` does - not stage these, so ``im_init`` must -- this is what makes ``im_pipeline`` - runnable from raw images, not just from pre-staged intermediates. + the same template serves every sim; ``config_mask.yaml`` is a staged copy + with the image-sim DES-table policy, and ``cfis`` is a symlink. Both are + what the downstream calibration and merge steps read from cwd. The mask + config is copied, not linked: a link into the sp_validation checkout or + spv's code snapshot breaks when that is removed or refreshed, and the run + dir should keep the mask it was calibrated with. ``tiles_{sim}.txt`` is + both the ShapePipe campaign's ``tile_list`` and the tile-ID file + ``params.py`` names for the found/missing-tile check. """ input: # Tracked so that editing the params template or mask config re-stages @@ -271,25 +364,38 @@ rule im_init: # silently leaving stale params.py behind after a grammar change). template=PARAMS_TEMPLATE, mask_src=os.path.join(SPV_REPO, MASK_CONFIG), + weight_product=( + [IMSIM["des_weight_product"]["path"]] + if "w_des" in IMSIM["w_cols"] + and IMSIM["des_weight_source"] == "external" + and IMSIM["des_weight_product"] else [] + ), output: params=f"{GRIDS_BASE}/{{sim}}/params.py", mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", + tiles=f"{GRIDS_BASE}/{{sim}}/tiles_{{sim}}.txt", params: - config_dir=CONFIG_DIR, - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + config_dir=SP_CONFIG_DIR, cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis", - sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles", - sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_exp", + tile_ids=" ".join(TILE_IDS), + source=IMSIM["des_weight_source"], + product_name=(IMSIM["des_weight_product"] or {}).get("name", "-"), + product_path=(IMSIM["des_weight_product"] or {}).get("path", "-"), + w_cols=IMSIM["w_cols"], + container: + SIF shell: - # cfis / input_tiles / input_exp are stable read-only symlinks (used by - # get_images, merge, extract); created here but not tracked as outputs, - # which snakemake will not accept for a symlink/directory. + # cfis is a stable read-only symlink (used by merge and extract); + # created here but not tracked as an output, which snakemake will not + # accept for a symlink to a directory. "mkdir -p $(dirname {output.params}) && " "cp {input.template} {output.params} && " - "ln -sf {input.mask_src} {output.mask} && " + "{_ENV_PREFIX} python {SPV_REPO}/scripts/calibration/stage_image_sim_config.py " + "--input {input.mask_src:q} --output {output.mask:q} " + "--w-cols {params.w_cols} --source {params.source:q} " + "--product-name {params.product_name:q} --product-path {params.product_path:q} && " "ln -sfT {params.config_dir} {params.cfis} && " - "ln -sfT {params.sim_tiles} {params.run_dir}/input_tiles && " - "ln -sfT {params.sim_exp} {params.run_dir}/input_exp" + "printf '%s\\n' {params.tile_ids} > {output.tiles}" rule im_pipeline: @@ -306,11 +412,9 @@ rule im_pipeline: # produces and aborts on a fresh grids_base. params=f"{GRIDS_BASE}/{{sim}}/params.py", output: - done=touch(f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done"), - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - psf=IMSIM["psf_model"], - n_smp=IMSIM["n_smp"], + record=f"{GRIDS_BASE}/{{sim}}/logs/shapepipe_campaign.yaml", + log: + f"{GRIDS_BASE}/{{sim}}/logs/sp_run.log", resources: mem_mb=16000, runtime=720, @@ -322,30 +426,25 @@ rule im_pipeline: "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " "-p {params.psf} -N {params.n_smp}" +def merged_cat(wc): + """This sim's merged catalogue. -rule im_merge: - """Merge per-tile ShapePipe catalogues into final_cat_{sim}.hdf5. + ShapePipe's own workflow (``merge_final_cats``, shapepipe #891) publishes it + under ``product/`` with the rest of the campaign's products, and that is the + only copy surviving ``clean_tile``. This workflow no longer merges -- the + ``im_merge`` rule that wrote one at the branch root was dropped once the + ShapePipe side gained the step. - ``create_final_cat.py`` lives in the ShapePipe repo/image; run in image_sims - mode (``-I``) it walks the per-tile output under the run directory. + The branch-root fallback is kept for a catalogue merged by hand or by an + older campaign. NOTHING PRODUCES IT any more, so a sim with neither copy + fails at DAG build on a missing input instead of silently re-merging -- + the honest outcome, since the campaign that should have published it did + not. """ - input: - tiles=expand( - f"{GRIDS_BASE}/{{{{sim}}}}/logs/pipeline_{{tile}}.done", - tile=TILE_IDS, - ), - output: - cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - container: - SIF - shell: - "cd {params.run_dir} && " - "{_ENV_PREFIX} python {CREATE_FINAL_CAT} " - "-I -m final_cat_{wildcards.sim}.hdf5 -i .. " - "-p cfis/final_cat.param -P {wildcards.sim} " - "-o n_tiles_final.txt -v" + prod = f"{GRIDS_BASE}/{wc.sim}/product/final_cat_{wc.sim}.hdf5" + if os.path.exists(prod): + return prod + return f"{GRIDS_BASE}/{wc.sim}/final_cat_{wc.sim}.hdf5" rule im_extract: @@ -353,17 +452,30 @@ rule im_extract: ``extract_info.py`` reads ``params.py`` from cwd and the merged catalogue, writing ``shape_catalog_comprehensive_{shape}``. + + Stages the catalogue itself rather than assuming a producer left it in the + right place: ``params_im_sim.py`` sets ``data_dir = "."`` and names + ``final_cat_{name}.hdf5`` with ``name`` from the directory basename, so the + file has to be reachable from the branch dir under exactly that name. A + catalogue published under ``product/`` is linked in; one already at the + root is left alone. """ input: - cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", + cat=merged_cat, params=f"{GRIDS_BASE}/{{sim}}/params.py", output: cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + staged=lambda wc: f"{GRIDS_BASE}/{wc.sim}/final_cat_{wc.sim}.hdf5", container: SIF shell: + # Compare resolved paths: when the catalogue is already at the branch + # root, source and destination are the same file and `ln -sf` would + # replace it with a link to itself. + '[ "$(readlink -f {input.cat})" = "$(readlink -f {params.staged})" ] ' + "|| ln -sf {input.cat} {params.staged}; " "cd {params.run_dir} && {_ENV_PREFIX} python {EXTRACT_INFO}" @@ -388,6 +500,33 @@ rule im_calibrate: "{_ENV_PREFIX} python {CALIBRATE} -s calibrate" +def campaign_records(): + """Per-branch evidence that the ShapePipe campaign finished, for ordering. + + WHICH file that is depends on which workflow ran the campaign, and both + shapes are in use: ShapePipe's unified workflow publishes + product/index/run_report.json, while the older sp_validation-driven runs + left n_tiles_final.txt beside the merged catalogue at the branch root. + Pick whichever exists, so a grid processed years apart still gets an edge. + + Nothing READS these -- im_mbias_config.py takes no input.* -- so the path + only has to name a file that really exists for that branch. A branch with + none contributes nothing rather than blocking the DAG on a file no rule + produces; its merged catalogue is already a declared input elsewhere. + """ + found = [] + for sim in SIMS: + for cand in ( + f"{GRIDS_BASE}/{sim}/product/index/run_report.json", + f"{GRIDS_BASE}/{sim}/product/n_tiles_final.txt", + f"{GRIDS_BASE}/{sim}/n_tiles_final.txt", + ): + if os.path.exists(cand): + found.append(cand) + break + return found + + rule im_mbias_config: """Assemble ``m_bias_config.yaml`` for the m-bias step: the manifest's shear/branch facts, this run's science knobs, and git/container provenance. @@ -397,11 +536,20 @@ rule im_mbias_config: cats=expand( f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS ), + # The per-branch ShapePipe campaign records, for provenance: the + # edge that says every campaign finished before this config was + # written. ShapePipe's own workflow publishes run_report.json with the + # products; the old logs/shapepipe_campaign.yaml was im_pipeline's + # output and is never written now that `sp run` drives the campaigns. + # Nothing READS these -- im_mbias_config.py takes no input.* -- so the + # path only has to name something the campaign really produced. + campaigns=campaign_records(), output: cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", params: grids_base=GRIDS_BASE, num=NUM, + sims_type=SIMS_TYPE, cat_name=f"shape_catalog_cut_{SHAPE}.fits", sif=SIF, shapepipe_repo=SHAPEPIPE_REPO, @@ -414,8 +562,15 @@ rule im_mbias_config: n_bootstrap=IMSIM["n_bootstrap"], pair_match=IMSIM["pair_match"], bootstrap_seed=IMSIM["bootstrap_seed"], - container: - SIF + # NO container: -- deliberately. This is snakemake's `script:` directive, + # which pickles the `snakemake` object on the host and unpickles it inside + # the job, so host and job need compatible snakemake versions. They are not: + # the host has 9.16.3 and the image 9.22.0, and the pickle fails with + # "Can't get attribute 'InputFiles' on module snakemake.io". Nothing here + # needs the image anyway -- the script imports only hashlib, os, re, + # subprocess and yaml, and reads git metadata from the checkouts, which is + # easier outside the container than in. The `shell:` rules keep theirs; + # they pass strings, not pickles. script: "../scripts/im_mbias_config.py" diff --git a/workflow/scripts/im_mbias_config.py b/workflow/scripts/im_mbias_config.py index cf6fdba20..9acc9c987 100644 --- a/workflow/scripts/im_mbias_config.py +++ b/workflow/scripts/im_mbias_config.py @@ -35,6 +35,26 @@ def _git(repo, *args): return None +def _repo_provenance(repo): + """Branch and commit of ``repo``. A code snapshot taken by ``spv run`` or + ``sp run`` has no .git; its ``snapshot.json`` records the checkout it was + copied from, including whether that checkout had uncommitted changes.""" + snap = os.path.join(repo, "snapshot.json") + if os.path.isfile(snap): + with open(snap) as fh: + info = yaml.safe_load(fh) or {} + return { + "branch": info.get("branch"), + "commit": info.get("head"), + "dirty": info.get("dirty"), + "snapshot_of": info.get("source"), + } + return { + "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _git(repo, "rev-parse", "HEAD"), + } + + def _sif_revision(): """``org.opencontainers.image.revision`` from the running image's OCI labels. @@ -84,16 +104,8 @@ def _sif_revision(): "output_path": params.results, "provenance": { "manifest_sha256": manifest_sha256, - "sp_validation": { - "branch": _git( - params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD" - ), - "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), - }, - "shapepipe": { - "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), - }, + "sp_validation": _repo_provenance(params.sp_validation_repo), + "shapepipe": _repo_provenance(params.shapepipe_repo), "container": { "sif": params.sif, "ghcr_revision": _sif_revision(), diff --git a/workflow/scripts/status_sims.py b/workflow/scripts/status_sims.py new file mode 100644 index 000000000..d4fe08d85 --- /dev/null +++ b/workflow/scripts/status_sims.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python3 +"""Disk-scan status report for image-sims ShapePipe campaigns. + +Not a DAG node (same reasoning as shapepipe/workflow/scripts/run_report.py: +a report that's part of the DAG would itself be poisoned by the failures it +must enumerate). Run any time, mid-run or after: + + workflow/bin/spv status -c sp_1p2z_grid_3.yaml # one campaign (sp run config) + workflow/bin/spv status # every sp_*.yaml in it + python workflow/scripts/status_sims.py # directly + +For each tile/exposure and each stage, a manifest means complete, a log +with no manifest means failed, and neither means not yet attempted -- the +same three-way read completeness.py's own contract establishes. + +A failed stage's log also carries why it failed: the runner counts, and the +lines completeness_sims.py scraped out of the shapepipe logs. Those are +reported per unit and grouped by reason, so one bad input among hundreds of +tiles is readable without opening any log by hand. +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +# Two layouts share this script. The NATIVE sp_validation prototype fits the +# PSF at tile level (tile_fake_psf) and runs ngmix as one call; the UNIFIED +# ShapePipe workflow (shapepipe #891) moves the PSF to the exposure level +# (exp_psf), chunks ngmix into tile_ngmix_1..N and merges the chunks +# (tile_merge_cats). Pass --layout to force one; `auto` reads what is on disk. +NATIVE_TILE_STAGES = [ + "tile_get_images", + "tile_uncompress", + "tile_find_exposures", + "tile_merge_headers", + "tile_detect", + "tile_fake_psf", + "tile_vignets", + "tile_ngmix", + "tile_make_cat", +] +NATIVE_EXP_STAGES = ["exp_get_images", "exp_split"] + +UNIFIED_TILE_STAGES = [ + "tile_get_images", + "tile_uncompress", + "tile_find_exposures", + "tile_merge_headers", + "tile_detect", + "tile_vignets", + "tile_ngmix", + "tile_merge_cats", + "tile_make_cat", +] +UNIFIED_EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] + +# Back-compat for anything importing these names. +TILE_STAGES = NATIVE_TILE_STAGES +EXP_STAGES = NATIVE_EXP_STAGES + +# The Gen-2 RUN_NAME suffixes (run_sp_tile_Git, run_sp_exp_Gie, ...) -- +# shorter and more familiar than the stage names for a wide table. +STAGE_ABBR = { + "tile_get_images": "Git", + "tile_uncompress": "Uz", + "tile_find_exposures": "Fe", + "tile_merge_headers": "Mh_exp", + "tile_detect": "Sx", + "tile_fake_psf": "fpsf", + "tile_vignets": "ViVi", + "tile_ngmix": "Ng", + "tile_merge_cats": "Ms", + "tile_make_cat": "Mc", + "exp_get_images": "Gie", + "exp_split": "Sp", + "exp_psf": "Psf", +} + +SYMBOL = {"complete": ".", "failed": "X", "missing": " "} + + +_CHUNK_RE = re.compile(r"^(tile_ngmix)_\d+$") + + +def unit_records(unit_dir: Path): + """(done, failed) stage-name sets for one unit. + + A reclaimed unit has had its manifests deleted, so clean_tile/ + clean_exposure's tombstone is read as the record of what completed -- it + absorbs every manifest verbatim for exactly this reason. Without this a + FINISHED campaign reports all zeros, because the more complete a unit is + the fewer manifests survive on disk. + """ + done, failed = set(), set() + mdir, ldir = unit_dir / "manifests", unit_dir / "logs" + if mdir.is_dir(): + done |= {p.stem for p in mdir.glob("*.json")} + if ldir.is_dir(): + # An EMPTY log means the job never ran, not that it failed: snakemake + # creates the declared log path even for a job its cone never reached. + # Counting those as failures turned one dead tile into a column of Xs + # across every downstream stage (run_report calls them "not run"). + failed |= {p.stem for p in ldir.glob("*.json") if p.stat().st_size > 0} + tomb = unit_dir / "cleaned.json" + if tomb.exists(): + try: + body = json.loads(tomb.read_text()) + except (OSError, ValueError): + body = {} + done |= set((body.get("manifests") or {})) + return done, failed - done + + +def unit_statuses(unit_dir: Path, stages: list, n_chunks: int) -> dict: + """stage -> complete | failed | missing, with the chunked ngmix collapsed + into its single `tile_ngmix` column (complete only when every chunk is).""" + done, failed = unit_records(unit_dir) + + chunks_done = {m for m in done if _CHUNK_RE.match(m)} + chunks_failed = {m for m in failed if _CHUNK_RE.match(m)} + + out = {} + for s in stages: + if s == "tile_ngmix" and n_chunks > 1: + if len(chunks_done) >= n_chunks: + out[s] = "complete" + elif chunks_failed or chunks_done: + out[s] = "failed" if chunks_failed else "missing" + else: + out[s] = "missing" + elif s in done: + out[s] = "complete" + elif s in failed: + out[s] = "failed" + else: + out[s] = "missing" + return out + + +def units_of(run_dir: Path, kind: str) -> list: + root = run_dir / kind + return sorted(p for p in root.glob("*/*") if p.is_dir()) if root.is_dir() else [] + + +def ngmix_chunks(run_dir: Path) -> int: + """How many chunks this run splits ngmix into: the most any one tile + records. 1 means the native single-call layout.""" + best = 1 + for unit_dir in units_of(run_dir, "tiles"): + done, failed = unit_records(unit_dir) + n = len({m for m in (done | failed) if _CHUNK_RE.match(m)}) + best = max(best, n) + return best + + +def detect_layout(run_dir: Path) -> str: + """'unified' or 'native'. + + Structural markers first, because they exist from the first minute of a + run: the native prototype keeps a plain JSON index inside the run dir, + while bin/sp puts its snakemake state in a `-state` sibling and + its index in a sqlite file under the products root. Stage-name sniffing is + the fallback, and on its own it mis-reads a young unified run as native -- + none of tile_ngmix_*, tile_merge_cats or exp_psf has been reached yet. + """ + if (run_dir / "index.json").is_file(): + return "native" + if run_dir.with_name(run_dir.name + "-state").is_dir(): + return "unified" + for name in ("product", "products"): + if (run_dir.parent / name / "index" / "run_index.sqlite").is_file(): + return "unified" + + for kind in ("tiles", "exp"): + for unit_dir in units_of(run_dir, kind): + done, failed = unit_records(unit_dir) + names = done | failed + if ( + any(_CHUNK_RE.match(m) for m in names) + or "tile_merge_cats" in names + or "exp_psf" in names + ): + return "unified" + if "tile_fake_psf" in names: + return "native" + return "native" + + +def stage_status(unit_dir: Path, stage: str) -> str: + """Kept for callers that ask about one stage; prefer unit_statuses.""" + return unit_statuses(unit_dir, [stage], 1)[stage] + + +# Group reasons that differ only in tile id, path or count, so N tiles failing +# the same way collapse to one line. +# Two segments minimum, so a count like "0/3" is not mistaken for a path. +_PATH_RE = re.compile(r"(?:/[^/\s]+){2,}") +_NUM_RE = re.compile(r"\d+") + + +def reason_shape(reason: str) -> str: + return _NUM_RE.sub("N", _PATH_RE.sub("", reason)) + + +def short_paths(reason: str) -> str: + """Absolute paths down to their file name: every run dir prefix is the + same, and the full lines are far too wide to scan.""" + return _PATH_RE.sub(lambda m: m.group(0).rsplit("/", 1)[-1], reason) + + +def stage_failures(unit_dir: Path, stage: str, n_chunks: int = 1) -> list: + """Why `stage` failed, as " found/expect" plus the scraped lines. + + Reads the log completeness_sims.py wrote. An unreadable or reason-less log + still counts as a failure -- it just cannot say more than that. + """ + logs = [unit_dir / "logs" / f"{stage}.json"] + if stage == "tile_ngmix" and n_chunks > 1: + # No tile_ngmix.json exists in the chunked layout; each chunk logs + # separately, and any of them can be the one that failed. + logs = sorted((unit_dir / "logs").glob("tile_ngmix_*.json")) + + out = [] + for log in logs: + try: + body = json.loads(log.read_text()) + except (OSError, ValueError): + out.append(f"{log.name}: log unreadable") + continue + tag = f"{log.stem}: " if len(logs) > 1 else "" + for rec in body.get("failures", []): + head = f"{tag}{rec.get('runner')}: {rec.get('found')}/{rec.get('expect')}" + reasons = rec.get("reasons") or [] + out.append(head if not reasons else f"{head} -- {reasons[0]}") + out += reasons[1:] + return out or ["no reason recorded"] + + +def report_failures(run_dir: Path, kind: str, stages: list, n_chunks: int = 1) -> None: + """Every failed unit with its reasons, then the same grouped by reason.""" + units = units_of(run_dir, kind) + + failed, groups = [], {} + for unit_dir in units: + statuses = unit_statuses(unit_dir, stages, n_chunks) + for s in stages: + if statuses[s] != "failed": + continue + reasons = stage_failures(unit_dir, s, n_chunks) + failed.append((unit_dir.name, s, reasons)) + for r in reasons: + key = (s, reason_shape(r)) + groups.setdefault(key, [r, []])[1].append(unit_dir.name) + + if not failed: + print(f"no failed {kind}") + print() + return + + n_units = len({u for u, _, _ in failed}) + print(f"{n_units} {kind} failed, {len(failed)} stage failures") + print() + for unit, stage, reasons in failed: + print(f" {unit:<12} {STAGE_ABBR[stage]}") + for r in reasons: + print(f" {short_paths(r)}") + print() + + # Only worth a second pass when a reason actually spans several units. + shared = {k: v for k, v in groups.items() if len(v[1]) > 1} + if not shared: + return + print(" by reason:") + for (stage, _), (reason, hit) in sorted( + shared.items(), key=lambda kv: -len(kv[1][1]) + ): + print(f" {len(hit):>4}x {STAGE_ABBR[stage]:<8} {short_paths(reason)}") + print(f" {', '.join(hit[:8])}" + (" ..." if len(hit) > 8 else "")) + print() + + +def report_finished(run_dir: Path, kind: str, stages: list, n_chunks: int = 1) -> None: + """--very-short: the abbreviated header, and ONE row below it -- how many + units have COMPLETED each step -- instead of one row per unit.""" + units = units_of(run_dir, kind) + n_cleaned = sum(1 for u in units if (u / "cleaned.json").exists()) + print(f"{len(units)} {kind}") + if not units: + return + if n_cleaned: + # Say it outright. The counts below come from the tombstones once a + # unit is reclaimed, so a FINISHED, fully cleaned campaign prints a row + # of full numbers and looks untouched -- the opposite of the pre- + # tombstone behaviour, where it printed zeros and looked emptied. + what = "store deleted" if n_cleaned == len(units) else "stores deleted" + print( + f" {n_cleaned}/{len(units)} RECLAIMED ({what}); the row below is " + f"what ran, read back from cleaned.json" + ) + + header = f"{'':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) + print(header) + print("-" * len(header)) + + counts = {s: 0 for s in stages} + for unit_dir in units: + statuses = unit_statuses(unit_dir, stages, n_chunks) + for s in stages: + if statuses[s] == "complete": + counts[s] += 1 + row = f"{'complete':<12}" + "".join(f"{counts[s]:>8}" for s in stages) + print(row) + + +def report_units( + run_dir: Path, kind: str, stages: list, *, short: bool = False, n_chunks: int = 1 +) -> None: + units = units_of(run_dir, kind) + if not units: + print(f"No {kind} units found under {run_dir / kind}") + return + + counts = {s: {"complete": 0, "failed": 0, "missing": 0} for s in stages} + + header = f"{'unit':<12}" + "".join(f"{STAGE_ABBR[s]:>8}" for s in stages) + print(header) + print("-" * len(header)) + + for unit_dir in units: + unit = unit_dir.name + row = f"{unit:<12}" + statuses = unit_statuses(unit_dir, stages, n_chunks) + for s in stages: + st = statuses[s] + counts[s][st] += 1 + row += f"{SYMBOL[st]:>8}" + print(row) + + # --short: just the per-unit table above, skip the aggregate counts below. + if short: + print() + return + + print() + print(f"{len(units)} {kind} total") + for s in stages: + c = counts[s] + print( + f" {STAGE_ABBR[s]:<8} complete={c['complete']:<4} failed={c['failed']:<4} " + f"not-yet-run={c['missing']}" + ) + print() + + +def final_cats(products_dir: Path) -> list: + """The published catalogues, on the products root (which is run_dir when + the run config sets no products_dir).""" + return sorted(products_dir.glob("tiles/*/*/final_cat-*.fits")) + + +def guess_products(run_dir: Path) -> Path: + """Where the catalogues are when only a run dir was given. + + The native layout publishes under the run root itself; the unified one + puts run/ and product/ side by side, so a bare `status_sims.py ` + would otherwise report `final_cat produced: 0` for a finished campaign. + """ + if final_cats(run_dir): + return run_dir + for name in ("product", "products"): + cand = run_dir.parent / name + if final_cats(cand): + return cand + return run_dir + + +def report_disk(run_dir: Path, enabled: bool) -> None: + """What the run directory still occupies -- the direct answer to "was this + cleaned up?", which the stage table alone cannot give.""" + if not enabled: + return + total = files = 0 + for dirpath, _, names in os.walk(run_dir): + for n in names: + try: + total += os.stat(os.path.join(dirpath, n)).st_size + files += 1 + except OSError: + pass + for unit, div in (("T", 1 << 40), ("G", 1 << 30), ("M", 1 << 20), ("K", 1 << 10)): + if total >= div: + size = f"{total / div:.1f}{unit}" + break + else: + size = f"{total}B" + print(f"run dir on disk: {size} in {files} files") + + +def report_merged(products_dir: Path, cats: list) -> None: + """The merged hdf5, the campaign's hand-off to sp_validation. + + Per-tile catalogues are only half the story: the sp_validation side starts + from ONE hdf5 per branch, so a campaign whose tiles are all green but whose + merge never ran is not actually finished. Checked against the per-tile + FITS, because a merge that silently picked up a subset is the failure worth + catching -- create_final_cat.py skips a tile it cannot find and says so + only in its own verbose output. + """ + merged = sorted(products_dir.glob("final_cat_*.hdf5")) + if not merged: + print( + f"merged hdf5: none in {products_dir} " + f"(run merge_final_cats, or create_final_cat.py -I -c )" + ) + return + + for path in merged: + try: + import h5py + except ImportError: + print( + f"merged hdf5: {path.name} ({path.stat().st_size / 1e6:.0f} MB)" + " -- install h5py to check its contents" + ) + return + try: + with h5py.File(path, "r") as fh: + groups = {} + + def walk(name, obj): + if isinstance(obj, h5py.Dataset): + groups[name] = obj.shape[0] + + fh.visititems(walk) + except OSError as exc: + print(f"merged hdf5: {path.name} UNREADABLE ({exc})") + continue + + n_tiles, n_rows = len(groups), sum(groups.values()) + # Row totals from the per-tile FITS headers only -- no pixel data read. + src_rows = 0 + for c in cats: + try: + from astropy.io import fits + + with fits.open(c, memmap=True) as hdus: + src_rows += hdus[1].header["NAXIS2"] + except Exception: + src_rows = -1 + break + + size = path.stat().st_size / 1e6 + note = "" + if len(cats) and n_tiles != len(cats): + note = f" MISMATCH: {len(cats)} final_cat files on disk" + elif src_rows >= 0 and n_rows != src_rows: + note = f" MISMATCH: {src_rows:,} rows in the per-tile catalogues" + print( + f"merged hdf5: {path.name} {n_tiles} tiles, {n_rows:,} rows, " + f"{size:.0f} MB{note}" + ) + + n_file = products_dir / "n_tiles_final.txt" + if n_file.exists(): + recorded = n_file.read_text().strip() + flag = "" if recorded == str(len(cats)) else f" MISMATCH: {len(cats)} on disk" + print(f"n_tiles_final.txt: {recorded}{flag}") + + +def _expand(config: dict) -> dict: + """Expand $name and ${name} in every value, from the top-level scalar keys, + repeatedly (as ShapePipe's run_config.py does for an sp run config).""" + scalars = { + k: v + for k, v in config.items() + if isinstance(v, (str, int, float)) and not isinstance(v, bool) + } + + def sub(value, depth=0): + if isinstance(value, str): + out = re.sub( + r"\$\{(\w+)\}|\$(\w+)", + lambda m: str(scalars.get(m.group(1) or m.group(2), m.group(0))), + value, + ) + return out if out == value or depth >= 5 else sub(out, depth + 1) + if isinstance(value, dict): + return {k: sub(v, depth) for k, v in value.items()} + if isinstance(value, list): + return [sub(v, depth) for v in value] + return value + + return {k: sub(v) for k, v in config.items()} + + +def run_dirs_from_config(path: Path): + """(run_dir, products_dir) from a run config's outputs: block; products_dir + defaults to run_dir, as in ShapePipe's workflow.""" + import yaml + + config = _expand(yaml.safe_load(Path(path).read_text()) or {}) + outputs = config.get("outputs") or {} + run_dir = outputs.get("run_dir") + if not run_dir or "$" in str(run_dir): + raise ValueError( + f"{path}: outputs.run_dir is unset or unexpanded ({run_dir!r})" + ) + return Path(run_dir), Path(outputs.get("products_dir") or run_dir) + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "run_dir", type=Path, nargs="?", help="run directory; omit when passing -c" + ) + p.add_argument( + "-c", + "--config", + type=Path, + help="ShapePipe run config (sp_*.yaml): read outputs.run_dir " + "and outputs.products_dir from it, $name/${name} expanded", + ) + p.add_argument( + "-s", + "--short", + action="store_true", + help="per-tile/per-exposure table only, no aggregate " + "'N units total' / per-stage count summary", + ) + p.add_argument( + "-S", + "--very-short", + action="store_true", + help="no per-unit table at all -- just 'N/M finished' " + "for tiles and exposures, once each (overrides -s)", + ) + p.add_argument( + "-p", + "--products", + type=Path, + help="products root, when it is neither the run dir nor a " + "product/ sibling of it", + ) + p.add_argument( + "--du", + action="store_true", + help="also measure the run directory on disk (a full walk; " + "slow on a live campaign, instant on a reclaimed one)", + ) + p.add_argument( + "--layout", + choices=("auto", "native", "unified"), + default="auto", + help="stage set to report: the sp_validation native " + "prototype, the unified ShapePipe workflow, or auto " + "(default) to read it off the run dir", + ) + args = p.parse_args(argv) + + if args.config: + try: + run_dir, products_dir = run_dirs_from_config(args.config) + except (OSError, ValueError) as exc: + p.error(f"{exc}") + elif args.run_dir: + run_dir = args.run_dir + products_dir = guess_products(run_dir) + else: + p.error("give a run directory, or -c with a run config") + + if args.products: + products_dir = args.products + + layout = args.layout if args.layout != "auto" else detect_layout(run_dir) + tile_stages = UNIFIED_TILE_STAGES if layout == "unified" else NATIVE_TILE_STAGES + exp_stages = UNIFIED_EXP_STAGES if layout == "unified" else NATIVE_EXP_STAGES + n_chunks = ngmix_chunks(run_dir) if layout == "unified" else 1 + + print(f"=== {run_dir} ===") + if products_dir != run_dir: + print(f"=== products: {products_dir} ===") + chunk_note = f", ngmix x{n_chunks}" if n_chunks > 1 else "" + print(f"=== layout: {layout}{chunk_note} ===") + + if args.very_short: + report_finished(run_dir, "tiles", tile_stages, n_chunks) + report_finished(run_dir, "exp", exp_stages) + cats = final_cats(products_dir) + print(f"final_cat produced: {len(cats)}") + report_merged(products_dir, cats) + report_disk(run_dir, args.du) + return 0 + + print() + print("--- tiles ---") + report_units(run_dir, "tiles", tile_stages, short=args.short, n_chunks=n_chunks) + print("--- exposures ---") + report_units(run_dir, "exp", exp_stages, short=args.short) + + if not args.short: + print("--- failures ---") + report_failures(run_dir, "tiles", tile_stages, n_chunks) + report_failures(run_dir, "exp", exp_stages) + + report_disk(run_dir, args.du) + cats = final_cats(products_dir) + print(f"final_cat produced: {len(cats)}") + if not args.short: + for f in cats: + print(f" {f}") + report_merged(products_dir, cats) + return 0 + + +if __name__ == "__main__": + sys.exit(main())