From 347f868cfb62ea27d367f4c07d6e50881a487a46 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 16 Feb 2026 08:19:09 +0100 Subject: [PATCH 1/3] Optional cut on ellipticity for calibration/metacal --- config/calibration/mask_v1.X.12.yaml | 113 +++++++++++++++++++++++ config/calibration/mask_v1.X.13.yaml | 113 +++++++++++++++++++++++ notebooks/calibrate_comprehensive_cat.py | 4 + src/sp_validation/basic.py | 16 ++++ 4 files changed, 246 insertions(+) create mode 100644 config/calibration/mask_v1.X.12.yaml create mode 100644 config/calibration/mask_v1.X.13.yaml diff --git a/config/calibration/mask_v1.X.12.yaml b/config/calibration/mask_v1.X.12.yaml new file mode 100644 index 00000000..39679dc7 --- /dev/null +++ b/config/calibration/mask_v1.X.12.yaml @@ -0,0 +1,113 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, type v1.X.12. + +# General parameters (can also given on command line) +params: + input_path: unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: equal + value: 0 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # ShapePipe flags + - col_name: IMAFLAGS_ISO + label: SP mask + kind: equal + value: 0 + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: r + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MOM_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +## Using columns in 'dat_ext' group (post-processing flags) +dat_ext: + + # Stars + - col_name: 4_Stars + label: "Stars" + kind: equal + value: False + + # Manual mask + - col_name: 8_Manual + label: "manual mask" + kind: equal + value: False + + # r-band footprint + - col_name: 64_r + label: "r-band imaging" + kind: equal + value: False + + # Maximask + - col_name: 1024_Maximask + label: "maximask" + kind: equal + value: False + + # Rough pointing coverage + - col_name: npoint3 + label: r"$n_{\rm pointing}$" + kind: greater_equal + value: 3 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Ellipticity cut + eps_max: 0.7 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: w diff --git a/config/calibration/mask_v1.X.13.yaml b/config/calibration/mask_v1.X.13.yaml new file mode 100644 index 00000000..179075cf --- /dev/null +++ b/config/calibration/mask_v1.X.13.yaml @@ -0,0 +1,113 @@ +# Config file for masking and calibration. +# Standard cuts without halo masks, type v1.X.13. + +# General parameters (can also given on command line) +params: + input_path: unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 3 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # ShapePipe flags + - col_name: IMAFLAGS_ISO + label: SP mask + kind: equal + value: 0 + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: r + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MOM_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +## Using columns in 'dat_ext' group (post-processing flags) +dat_ext: + + # Stars + - col_name: 4_Stars + label: "Stars" + kind: equal + value: False + + # Manual mask + - col_name: 8_Manual + label: "manual mask" + kind: equal + value: False + + # r-band footprint + - col_name: 64_r + label: "r-band imaging" + kind: equal + value: False + + # Maximask + - col_name: 1024_Maximask + label: "maximask" + kind: equal + value: False + + # Rough pointing coverage + - col_name: npoint3 + label: r"$n_{\rm pointing}$" + kind: greater_equal + value: 3 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.707 + gal_rel_size_max: 3 + + # Ellipticity cut + eps_max: 0.7 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean + global_R_weight: w diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index 404a7bec..dbf20634 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -90,6 +90,9 @@ # Call metacal cm = config["metacal"] +# Cut on ellipticity +eps_max = cm.get("eps_max", None) + # %% gal_metacal = metacal( dat, @@ -99,6 +102,7 @@ rel_size_min=cm["gal_rel_size_min"], rel_size_max=cm["gal_rel_size_max"], size_corr_ell=cm["gal_size_corr_ell"], + eps_max=eps_max, sigma_eps=cm["sigma_eps_prior"], global_R_weight=cm["global_R_weight"], col_2d=False, diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 8eed0eca..4d3e8afe 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -52,6 +52,8 @@ class metacal: relative size minimum rel_size_max : float, optional, default=3.0 relative size maximum + eps_max: float, optional + maximum ellipticiy (absolute value), default is ``None`` size_corr_ell : bool, optional, default=True global_R_weight : str, optional, weight column name for global response matrix; default is ``None`` @@ -78,6 +80,7 @@ def __init__( snr_max=500, rel_size_min=0.5, rel_size_max=3.0, + eps_max=None, size_corr_ell=True, global_R_weight=None, sigma_eps=0.34, @@ -93,12 +96,14 @@ def __init__( self._snr_max = snr_max self._rel_size_min = rel_size_min self._rel_size_max = rel_size_max + self._eps_max = eps_max self._size_corr_ell = size_corr_ell if verbose: print( f'Metacal cuts: {snr_min} self._eps_max + print(f"MKDEBUG cut on ellipticity of {self._eps_max}") + print(f"max eps after cut = {max(eps[mask_eps])}") + else: + mask_eps = True + mask_tmp = ( (data['flag'] == 0) & (Tr_tmp / Tpsf > self._rel_size_min) & (Tr_tmp / Tpsf < self._rel_size_max) & (snr_flux > self._snr_min) & (snr_flux < self._snr_max) + & mask_eps ) # Take care of rotated version From a45a9a9047b2b8a1047c4cf2c40de547e8be9a37 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 16 Feb 2026 08:19:26 +0100 Subject: [PATCH 2/3] fixed redshift distr entry --- src/sp_validation/cosmo_val.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 316de53e..5f5e5af6 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -2870,7 +2870,8 @@ def calculate_pseudo_cl_onecovariance(self): else: mask_path = self.cc[ver]['shear']['mask'] - redshift_distr_path = os.path.join(self.data_base_dir, self.cc[ver]['shear']['redshift_distr']) + #redshift_distr_path = os.path.join(self.data_base_dir, self.cc[ver]['shear']['redshift_distr']) + redshift_distr_path = self.cc[ver]['shear']['redshift_distr'] config_path = os.path.join(out_dir, f"config_onecov_{ver}.ini") From 2a9777f9acfca454eb92ef09781e5e2e4b3a45af Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 18 Feb 2026 08:56:15 +0100 Subject: [PATCH 3/3] print shear matrix from FITS header: added txt output; remove debug msg for ell cuts --- notebooks/cosmo_val/cat_config.yaml | 90 ++++++++++++++++++++++++++++ notebooks/cosmo_val/run_cosmo_val.py | 2 +- notebooks/demo_add_bands_to_empty.py | 6 +- notebooks/demo_apply_hsp_masks.py | 4 +- scripts/print_matrix_from_fits.py | 45 ++++++++++++-- src/sp_validation/basic.py | 4 +- 6 files changed, 137 insertions(+), 14 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 7a2bad29..fdfe4ed7 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -868,6 +868,96 @@ SP_v1.4.11.3: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.12.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: lightblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: ^ + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.13.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: h + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.11.3_ecut07: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index 2fae1705..517fa99c 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -89,8 +89,8 @@ max_sep_int=300, nbins_int=100, npatch=256, - var_method="jackknife", ) +#var_method="jackknife", # %% cv.plot_cosebis( diff --git a/notebooks/demo_add_bands_to_empty.py b/notebooks/demo_add_bands_to_empty.py index 14788826..e7cdf80e 100644 --- a/notebooks/demo_add_bands_to_empty.py +++ b/notebooks/demo_add_bands_to_empty.py @@ -43,7 +43,7 @@ # Set parameters base = "unions_shapepipe_comprehensive_struc_empty_ugriz" year = 2024 -ver = "v1.5.c" +ver = "v1.6.c" obj._params = {} @@ -62,14 +62,14 @@ base_keys = ["MAGERR_GAAP", "FLUX_GAAP", "FLUXERR_GAAP", "FLAG_GAAP", "MAG_LIM"] keys_mag = [f"MAG_GAAP_0p7_{band}" for band in bands] for base_key in base_keys: - keys_mag.extend([f"_{base_key}_{band}" for band in bands]) + keys_mag.extend([f"{base_key}_{band}" for band in bands]) keys = [ "EXTINCTION", "MP_NAME", "Z_B", "Z_B_MIN", - "Z_B_MAX" + "Z_B_MAX", "T_B", "ODDS", ] + keys_mag diff --git a/notebooks/demo_apply_hsp_masks.py b/notebooks/demo_apply_hsp_masks.py index a264bb1d..1ea07092 100644 --- a/notebooks/demo_apply_hsp_masks.py +++ b/notebooks/demo_apply_hsp_masks.py @@ -57,13 +57,13 @@ # + # Set parameters base = "unions_shapepipe_comprehensive" -ver = "v1.3.c" +ver = "v1.6.c" if ver == "v1.3.c": year = 2022 else: year = 2024 -ver_maj = "v1.5" +ver_maj = "v1.6" obj._params["input_path"] = f"{base}_{year}_{ver_maj}.c.hdf5" obj._params["output_path"] = f"{base}_struc_{year}_{ver_maj}.c.hdf5" diff --git a/scripts/print_matrix_from_fits.py b/scripts/print_matrix_from_fits.py index 0b284b19..f8ac873c 100755 --- a/scripts/print_matrix_from_fits.py +++ b/scripts/print_matrix_from_fits.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Read matrix entries from FITS header and print in tex or pdf format.""" +"""Read matrix entries from FITS header and print in txt, tex, or pdf format.""" from astropy.io import fits import numpy as np @@ -64,6 +64,34 @@ def get_matrix_from_header(header, prefix): return matrix +def matrix_to_txt(matrix, prec): + """Convert matrix to ASCII format. + + Parameters + ---------- + matrix : numpy.ndarray + 2x2 matrix + prec : int + number of significant digits + + Returns + ------- + str + ASCII-formatted matrix string + + """ + col_widths = [] + for j in range(2): + col_widths.append(max(len(f"{matrix[i, j]:.{prec}g}") for i in range(2))) + + lines = [] + for i in range(2): + vals = [f"{matrix[i, j]:.{prec}g}".rjust(col_widths[j]) for j in range(2)] + lines.append("[ " + " ".join(vals) + " ]") + + return "\n".join(lines) + + def matrix_to_tex(matrix, prec, exp_thresh): """Convert matrix to LaTeX format.""" rows = [] @@ -147,7 +175,7 @@ def main(): help_strings = { "input": "Input FITS file", "prefix": "Prefix for matrix entries (default: {})", - "format": "Output format: tex or pdf (default: {})", + "format": "Output format: txt, tex or pdf (default: {})", "prec": "Number of significant digits (default: {})", "exp_thresh": "Exponent threshold for scientific notation (default: {})", "output": "Output file path (extension added based on format)", @@ -159,8 +187,8 @@ def main(): if options["input"] is None: raise ValueError("Input FITS file (-i) is required") - if options["format"] not in ["tex", "pdf"]: - raise ValueError("Format must be 'tex' or 'pdf'") + if options["format"] not in ["txt", "tex", "pdf"]: + raise ValueError("Format must be 'txt', 'tex', or 'pdf'") # Read FITS header with fits.open(options["input"]) as hdu: @@ -177,7 +205,14 @@ def main(): if options["output"]: output_path = f"{options['output']}.{options['format']}" - if options["format"] == "tex": + if options["format"] == "txt": + txt_content = matrix_to_txt(matrix, options["prec"]) + print(txt_content) + if output_path: + with open(output_path, "w") as f: + f.write(txt_content + "\n") + print(f"Written to {output_path}") + elif options["format"] == "tex": full_tex = r"""\documentclass[preview,border=2pt]{article} \usepackage{amsmath} \begin{document} diff --git a/src/sp_validation/basic.py b/src/sp_validation/basic.py index 4d3e8afe..4c266045 100644 --- a/src/sp_validation/basic.py +++ b/src/sp_validation/basic.py @@ -390,9 +390,7 @@ def _masking_gal(self): eps = np.sqrt( data["g1"] ** 2 + data["g2"] ** 2 ) - mask_eps = eps > self._eps_max - print(f"MKDEBUG cut on ellipticity of {self._eps_max}") - print(f"max eps after cut = {max(eps[mask_eps])}") + mask_eps = eps < self._eps_max else: mask_eps = True