diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 958b87e..9f7353e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,6 @@ on: push: tags: - 'v*.*.*' - - 'test*' workflow_dispatch: permissions: diff --git a/src/pmotools/pmo_builder/metatable_to_pmo.py b/src/pmotools/pmo_builder/metatable_to_pmo.py index 4162c22..6169646 100644 --- a/src/pmotools/pmo_builder/metatable_to_pmo.py +++ b/src/pmotools/pmo_builder/metatable_to_pmo.py @@ -157,7 +157,7 @@ def library_sample_info_table_to_pmo( library_prep_plate_position_col, meta_json, copy_contents, - "specimen_name", + "library_sample_name", "library_prep_plate_info", ) meta_json = add_parasite_density_info( @@ -166,7 +166,7 @@ def library_sample_info_table_to_pmo( meta_json, copy_contents, "library_sample_name", - entry_name="parasite_density_info", + entry_name="qpcr_parasite_density_info", ) # listify columns that contain values that could be list, are delimited by the argument list_values_library_values_delimiter primitives = (int, float, str, bool, complex) @@ -494,7 +494,7 @@ def add_plate_info( plate_position_col, meta_json, df, - specimen_name_col, + match_col, entry_name="plate_info", ): if all( @@ -531,7 +531,7 @@ def add_plate_info( ) from e for row in meta_json: - content_row = df[df[specimen_name_col] == row[specimen_name_col]] + content_row = df[df[match_col] == row[match_col]] plate_name_val = content_row[plate_name_col].iloc[0] if plate_name_col else None plate_row_val = ( content_row[plate_row_col].iloc[0].upper() if plate_row_col else None @@ -560,7 +560,7 @@ def add_parasite_density_info( parasite_density_method_col, meta_json, df, - specimen_name_col, + match_col, entry_name, ): density_method_pairs = [] @@ -608,7 +608,7 @@ def add_parasite_density_info( # Add parasite density info to meta_json for row in meta_json: - content_row = df[df[specimen_name_col] == row[specimen_name_col]] + content_row = df[df[match_col] == row[match_col]] density_infos = [] for density_col, method_col in density_method_pairs: density_val = content_row[density_col].iloc[0] if density_col else None diff --git a/src/pmotools/pmo_builder/mhap_table_to_pmo.py b/src/pmotools/pmo_builder/mhap_table_to_pmo.py index be0ae34..fb00da7 100644 --- a/src/pmotools/pmo_builder/mhap_table_to_pmo.py +++ b/src/pmotools/pmo_builder/mhap_table_to_pmo.py @@ -26,6 +26,13 @@ def mhap_table_to_pmo( masking_delim: str = ",", microhaplotype_name_col: str | None = None, pseudocigar_col: str | None = None, + pseudocigar_chrom_col: str | None = None, + pseudocigar_start_col: str | None = None, + pseudocigar_end_col: str | None = None, + pseudocigar_ref_seq_col: str | None = None, + pseudocigar_strand_col: str | None = None, + pseudocigar_genome_id: int | None = None, + pseudocigar_generation_description_col: str | None = None, quality_col: str | None = None, additional_representative_mhap_cols: list | None = None, additional_mhap_detected_cols: list | None = None, @@ -73,6 +80,20 @@ def mhap_table_to_pmo( :type microhaplotype_name_col: str, optional :param pseudocigar_col: the name of the column containing a pseudocigar for the microhaplotype :type pseudocigar_col: str, optional + :param pseudocigar_chrom_col: column for the pseudocigar ref_loc chromosome; defaults to chrom_col + :type pseudocigar_chrom_col: str, optional + :param pseudocigar_start_col: column for the pseudocigar ref_loc start (required if pseudocigar_col is set) + :type pseudocigar_start_col: str, optional + :param pseudocigar_end_col: column for the pseudocigar ref_loc end (required if pseudocigar_col is set) + :type pseudocigar_end_col: str, optional + :param pseudocigar_ref_seq_col: optional column for the pseudocigar ref_loc reference sequence + :type pseudocigar_ref_seq_col: str, optional + :param pseudocigar_strand_col: optional column for the pseudocigar ref_loc strand + :type pseudocigar_strand_col: str, optional + :param pseudocigar_genome_id: genome id for the pseudocigar ref_loc; defaults to genome_id + :type pseudocigar_genome_id: int, optional + :param pseudocigar_generation_description_col: optional column describing how the pseudocigar was generated + :type pseudocigar_generation_description_col: str, optional :param quality_col: the name of the column containing the ANSI FASTQ per-base quality score for this sequence :type quality_col: str, optional :param additional_representative_mhap_cols: additional columns to add to the representative microhaplotypes table @@ -100,6 +121,13 @@ def mhap_table_to_pmo( masking_delim=masking_delim, microhaplotype_name_col=microhaplotype_name_col, pseudocigar_col=pseudocigar_col, + pseudocigar_chrom_col=pseudocigar_chrom_col, + pseudocigar_start_col=pseudocigar_start_col, + pseudocigar_end_col=pseudocigar_end_col, + pseudocigar_ref_seq_col=pseudocigar_ref_seq_col, + pseudocigar_strand_col=pseudocigar_strand_col, + pseudocigar_genome_id=pseudocigar_genome_id, + pseudocigar_generation_description_col=pseudocigar_generation_description_col, quality_col=quality_col, additional_representative_mhap_cols=additional_representative_mhap_cols, ) @@ -160,6 +188,13 @@ def create_representative_microhaplotype_dict( masking_delim: str = ",", microhaplotype_name_col: str | None = None, pseudocigar_col: str | None = None, + pseudocigar_chrom_col: str | None = None, + pseudocigar_start_col: str | None = None, + pseudocigar_end_col: str | None = None, + pseudocigar_ref_seq_col: str | None = None, + pseudocigar_strand_col: str | None = None, + pseudocigar_genome_id: int | None = None, + pseudocigar_generation_description_col: str | None = None, quality_col: str | None = None, additional_representative_mhap_cols: list[str] | None = None, ): @@ -198,6 +233,20 @@ def create_representative_microhaplotype_dict( :type microhaplotype_name_col: str, optional :param pseudocigar_col: the name of the column containing a pseudocigar for the microhaplotype :type pseudocigar_col: str, optional + :param pseudocigar_chrom_col: column for the pseudocigar ref_loc chromosome; defaults to chrom_col + :type pseudocigar_chrom_col: str, optional + :param pseudocigar_start_col: column for the pseudocigar ref_loc start (required if pseudocigar_col is set) + :type pseudocigar_start_col: str, optional + :param pseudocigar_end_col: column for the pseudocigar ref_loc end (required if pseudocigar_col is set) + :type pseudocigar_end_col: str, optional + :param pseudocigar_ref_seq_col: optional column for the pseudocigar ref_loc reference sequence + :type pseudocigar_ref_seq_col: str, optional + :param pseudocigar_strand_col: optional column for the pseudocigar ref_loc strand + :type pseudocigar_strand_col: str, optional + :param pseudocigar_genome_id: genome id for the pseudocigar ref_loc; defaults to genome_id + :type pseudocigar_genome_id: int, optional + :param pseudocigar_generation_description_col: optional column describing how the pseudocigar was generated + :type pseudocigar_generation_description_col: str, optional :param quality_col: the name of the column containing the ANSI FASTQ per-base quality score for this sequence :type quality_col: str, optional :param additional_representative_mhap_cols: additional columns to add to the representative microhaplotypes table @@ -211,6 +260,23 @@ def create_representative_microhaplotype_dict( microhaplotype_table, additional_representative_mhap_cols ) + # the pseudocigar ref_loc shares the chromosome / genome with the + # microhaplotype location by default, but each may use its own column + pseudocigar_chrom_col = ( + pseudocigar_chrom_col if pseudocigar_chrom_col else chrom_col + ) + ps_genome_id = ( + pseudocigar_genome_id if pseudocigar_genome_id is not None else genome_id + ) + if pseudocigar_col and not ( + pseudocigar_chrom_col and pseudocigar_start_col and pseudocigar_end_col + ): + raise ValueError( + "pseudocigar_col is set, so a Pseudocigar ref_loc must be " + "constructable: set a chromosome (pseudocigar_chrom_col or " + "chrom_col), pseudocigar_start_col, and pseudocigar_end_col." + ) + def get_if_present(row, col): return row[col] if col and pd.notna(row[col]) else None @@ -263,6 +329,12 @@ def warn_if_duplicated_seqs(df, target_col, seq_col): alt_annotations_col, microhaplotype_name_col, pseudocigar_col, + pseudocigar_chrom_col, + pseudocigar_start_col, + pseudocigar_end_col, + pseudocigar_ref_seq_col, + pseudocigar_strand_col, + pseudocigar_generation_description_col, quality_col, ] masking_cols = [ @@ -319,7 +391,34 @@ def warn_if_duplicated_seqs(df, target_col, seq_col): if val := get_if_present(row, microhaplotype_name_col): mhap["microhaplotype_name"] = val if val := get_if_present(row, pseudocigar_col): - mhap["pseudo_cigar"] = val + if not ( + pd.notna(row[pseudocigar_chrom_col]) + and pd.notna(row[pseudocigar_start_col]) + and pd.notna(row[pseudocigar_end_col]) + ): + raise ValueError( + f"pseudocigar present for target {target}, seq " + f"{row[seq_col]} but its ref_loc chrom/start/end " + "is missing" + ) + ref_loc = { + "genome_id": ps_genome_id, + "chrom": row[pseudocigar_chrom_col], + "start": row[pseudocigar_start_col], + "end": row[pseudocigar_end_col], + } + if pseudocigar_ref_seq_col and pd.notna(row[pseudocigar_ref_seq_col]): + ref_loc["ref_seq"] = row[pseudocigar_ref_seq_col] + if pseudocigar_strand_col and pd.notna(row[pseudocigar_strand_col]): + ref_loc["strand"] = row[pseudocigar_strand_col] + pseudocigar = {"pseudocigar_seq": val, "ref_loc": ref_loc} + if pseudocigar_generation_description_col and pd.notna( + row[pseudocigar_generation_description_col] + ): + pseudocigar["pseudocigar_generation_description"] = row[ + pseudocigar_generation_description_col + ] + mhap["pseudocigar"] = pseudocigar if val := get_if_present(row, quality_col): mhap["quality"] = val if additional_representative_mhap_cols: diff --git a/src/pmotools/pmo_builder/panel_information_to_pmo.py b/src/pmotools/pmo_builder/panel_information_to_pmo.py index 949fd8c..563307f 100644 --- a/src/pmotools/pmo_builder/panel_information_to_pmo.py +++ b/src/pmotools/pmo_builder/panel_information_to_pmo.py @@ -332,8 +332,8 @@ def build_target_info_dict( fwd_primer_dict["location"] = { "genome_id": genome_id, "chrom": row[chrom_col], - "end": int(row[forward_primers_start_col]), - "start": int(row[forward_primers_end_col]), + "start": int(row[forward_primers_start_col]), + "end": int(row[forward_primers_end_col]), } if strand_col and pd.notna(row[strand_col]): fwd_primer_dict["location"]["strand"] = row[strand_col] diff --git a/src/pmotools/pmo_engine/pmo_exporter.py b/src/pmotools/pmo_engine/pmo_exporter.py index 361ebd7..a2a952c 100644 --- a/src/pmotools/pmo_engine/pmo_exporter.py +++ b/src/pmotools/pmo_engine/pmo_exporter.py @@ -121,10 +121,14 @@ def export_specimen_meta_table(pmodata, separator: str = ",") -> pd.DataFrame: for specimen in pmodata["specimen_info"]: export_row = {} for key, value in specimen.items(): - if "project_id" == key: + if "project_id" == key and "project_info" in pmodata: export_row["project_name"] = pmodata["project_info"][value][ "project_name" ] + elif "project_id" == key: + # project_info is optional as of schema v1.1.0; keep the raw id + # rather than failing to resolve a name that isn't available + export_row[key] = value elif PMOExporter._is_primitive(value): export_row[key] = value elif PMOExporter._is_primitive_list(value): @@ -145,10 +149,14 @@ def export_library_sample_meta_table(pmodata, separator: str = ",") -> pd.DataFr for library_sample in pmodata["library_sample_info"]: export_row = {} for key, value in library_sample.items(): - if "sequencing_info_id" == key: + if "sequencing_info_id" == key and "sequencing_info" in pmodata: export_row["sequencing_info_name"] = pmodata["sequencing_info"][ value ]["sequencing_info_name"] + elif "sequencing_info_id" == key: + # sequencing_info is optional as of schema v1.1.0; keep the raw id + # rather than failing to resolve a name that isn't available + export_row[key] = value elif "specimen_id" == key: export_row["specimen_name"] = pmodata["specimen_info"][value][ "specimen_name" @@ -451,6 +459,7 @@ def write_bed_locs(bed_locs: list[BedLoc], fnp, add_header: bool = False): ] ) ) + f.write("\n") for bed_loc in bed_locs: f.write( "\t".join( @@ -540,11 +549,10 @@ def extract_panels_insert_bed_loc( :param sort_output: whether to sort output by genomic location :return: a list of target inserts, with named tuples with fields: chrom, start, end, name, score, strand, ref_seq, extra_info """ - bed_loc_out = {} + bed_loc_out = [] if select_panel_ids is None: select_panel_ids = list(range(len(pmodata["panel_info"]))) for panel_id in select_panel_ids: - bed_loc_out_per_panel = [] for reaction_id in range(len(pmodata["panel_info"][panel_id]["reactions"])): for target_id in pmodata["panel_info"][panel_id]["reactions"][ reaction_id @@ -589,7 +597,7 @@ def extract_panels_insert_bed_loc( if "ref_seq" not in tar["insert_location"] else tar["insert_location"]["ref_seq"] ) - bed_loc_out_per_panel.append( + bed_loc_out.append( BedLoc( tar["insert_location"]["chrom"], tar["insert_location"]["start"], @@ -602,12 +610,8 @@ def extract_panels_insert_bed_loc( extra_info, ) ) - if sort_output: - return sorted( - bed_loc_out_per_panel, - key=lambda bed: (bed.chrom, bed.start, bed.end), - ) - bed_loc_out[panel_id] = bed_loc_out_per_panel + if sort_output: + return sorted(bed_loc_out, key=lambda bed: (bed.chrom, bed.start, bed.end)) return bed_loc_out @staticmethod diff --git a/src/pmotools/pmo_engine/pmo_processor.py b/src/pmotools/pmo_engine/pmo_processor.py index cef95fb..1e38ca5 100644 --- a/src/pmotools/pmo_engine/pmo_processor.py +++ b/src/pmotools/pmo_engine/pmo_processor.py @@ -20,6 +20,10 @@ def get_index_key_of_bioinformatics_run_names(pmodata): :return: a dictionary of indexes keyed by bioinformatics_run_name """ ret = {} + # bioinformatics_run_info is optional as of schema v1.1.0; a PMO without it + # simply has no run-name to index mapping + if "bioinformatics_run_info" not in pmodata: + return ret for idx, bioinformatics_run in enumerate(pmodata["bioinformatics_run_info"]): ret[bioinformatics_run["bioinformatics_run_name"]] = idx return ret @@ -646,8 +650,6 @@ def filter_pmo_by_library_sample_ids(pmodata, library_sample_ids: set[int]): if "targeted_genomes" in pmodata: pmo_out["targeted_genomes"] = copy.deepcopy(pmodata["targeted_genomes"]) - if "read_counts_by_stage" in pmodata: - pmo_out["read_counts_by_stage"] = [] # need to update read_counts_by_stage, library_sample_info, specimen_info, detected_microhaplotypes # specimen_info @@ -703,6 +705,7 @@ def filter_pmo_by_library_sample_ids(pmodata, library_sample_ids: set[int]): pmo_out["detected_microhaplotypes"].append(new_detected_microhaplotypes) # read_counts_by_stage if "read_counts_by_stage" in pmodata: + pmo_out["read_counts_by_stage"] = [] for read_count in pmodata["read_counts_by_stage"]: new_read_count = { "read_counts_by_library_sample_by_stage": [], @@ -725,6 +728,7 @@ def filter_pmo_by_library_sample_ids(pmodata, library_sample_ids: set[int]): ]["library_sample_id"] = library_id_index_key[ sample["library_sample_id"] ] + pmo_out["read_counts_by_stage"].append(new_read_count) return pmo_out @staticmethod @@ -884,11 +888,12 @@ def filter_pmo_by_target_ids(pmodata, target_ids: set[int]): pmo_out["representative_microhaplotypes"]["targets"] ) # update new target_id index - microhap_info["target_id"] = target_info_index_key[ + microhap_info_copy = copy.deepcopy(microhap_info) + microhap_info_copy["target_id"] = target_info_index_key[ microhap_info["target_id"] ] pmo_out["representative_microhaplotypes"]["targets"].append( - copy.deepcopy(microhap_info) + microhap_info_copy ) # representative_microhaplotypes pmo_out["detected_microhaplotypes"] = [] @@ -908,10 +913,11 @@ def filter_pmo_by_target_ids(pmodata, target_ids: set[int]): for target in sample["target_results"]: if target["mhaps_target_id"] in mhaps_target_id_new_key: # update with new mhaps_target_id id - target["mhaps_target_id"] = mhaps_target_id_new_key[ + target_copy = copy.deepcopy(target) + target_copy["mhaps_target_id"] = mhaps_target_id_new_key[ target["mhaps_target_id"] ] - new_sample["target_results"].append(copy.deepcopy(target)) + new_sample["target_results"].append(target_copy) new_detected_microhaplotypes["library_samples"].append(new_sample) pmo_out["detected_microhaplotypes"].append(new_detected_microhaplotypes) @@ -938,11 +944,12 @@ def filter_pmo_by_target_ids(pmodata, target_ids: set[int]): for target in sample["read_counts_for_targets"]: if target["target_id"] in target_ids: # update with new target_id index - target["target_id"] = target_info_index_key[ + target_copy = copy.deepcopy(target) + target_copy["target_id"] = target_info_index_key[ target["target_id"] ] new_samples["read_counts_for_targets"].append( - copy.deepcopy(target) + target_copy ) new_read_counts_by_bioid[ "read_counts_by_library_sample_by_stage" @@ -1073,23 +1080,29 @@ def extract_from_pmo_with_read_filter(pmodata, read_filter: float): pmo_out = { "pmo_header": copy.deepcopy(pmodata["pmo_header"]), "panel_info": copy.deepcopy(pmodata["panel_info"]), - "sequencing_info": copy.deepcopy(pmodata["sequencing_info"]), "target_info": copy.deepcopy(pmodata["target_info"]), "specimen_info": copy.deepcopy(pmodata["specimen_info"]), "library_sample_info": copy.deepcopy(pmodata["library_sample_info"]), - "project_info": copy.deepcopy(pmodata["project_info"]), - "targeted_genomes": copy.deepcopy(pmodata["targeted_genomes"]), "representative_microhaplotypes": copy.deepcopy( pmodata["representative_microhaplotypes"] ), - "bioinformatics_methods_info": copy.deepcopy( - pmodata["bioinformatics_methods_info"] - ), - "bioinformatics_run_info": copy.deepcopy( - pmodata["bioinformatics_run_info"] - ), "detected_microhaplotypes": [], } + # only copy optional sections if they are present (no longer required by the schema) + if "sequencing_info" in pmodata: + pmo_out["sequencing_info"] = copy.deepcopy(pmodata["sequencing_info"]) + if "project_info" in pmodata: + pmo_out["project_info"] = copy.deepcopy(pmodata["project_info"]) + if "targeted_genomes" in pmodata: + pmo_out["targeted_genomes"] = copy.deepcopy(pmodata["targeted_genomes"]) + if "bioinformatics_methods_info" in pmodata: + pmo_out["bioinformatics_methods_info"] = copy.deepcopy( + pmodata["bioinformatics_methods_info"] + ) + if "bioinformatics_run_info" in pmodata: + pmo_out["bioinformatics_run_info"] = copy.deepcopy( + pmodata["bioinformatics_run_info"] + ) # if has optional read_counts_by_stage then add as well # if does contain, @todo consider updating with new counts now that a filter has been applied diff --git a/src/pmotools/pmo_engine/pmo_reader.py b/src/pmotools/pmo_engine/pmo_reader.py index 91a6e46..deb7930 100644 --- a/src/pmotools/pmo_engine/pmo_reader.py +++ b/src/pmotools/pmo_engine/pmo_reader.py @@ -394,10 +394,11 @@ def combine_multiple_pmos(pmos: list[dict]): pmo["bioinformatics_run_info"] ): bioinformatics_run_info_copy = copy.deepcopy(bioinformatics_run_info) + # remap using the run's own methods id, not its position index bioinformatics_run_info_copy[ "bioinformatics_methods_id" ] = bioinformatics_methods_info_old_index_key[pmo_index][ - bioinformatics_run_info_index + bioinformatics_run_info_copy["bioinformatics_methods_id"] ] if "bioinformatics_run_info" not in pmo_out: pmo_out["bioinformatics_run_info"] = [] @@ -501,8 +502,13 @@ def combine_multiple_pmos(pmos: list[dict]): new_mhaps_target_index = len( pmo_out["representative_microhaplotypes"]["targets"] ) + new_rep_target = copy.deepcopy(representative_microhaplotypes) + # remap the new target's target_id to the combined target_info + new_rep_target["target_id"] = target_info_old_index_key[pmo_index][ + new_rep_target["target_id"] + ] pmo_out["representative_microhaplotypes"]["targets"].append( - copy.deepcopy(representative_microhaplotypes) + new_rep_target ) representative_microhaplotypes_old_index_key[pmo_index][ representative_microhaplotypes_index diff --git a/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py b/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py index bfd458b..93f8144 100755 --- a/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py +++ b/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py @@ -91,7 +91,7 @@ def get_parser() -> argparse.ArgumentParser: "--default_base_col_names", type=str, required=False, - default="library_sample_name,target_name,mhap_id", + default="library_sample_name,target_name,seq", help="default base column names, must be length 3", ) return parser diff --git a/tests/data/minimum_Furstenau2025_PMO.json.gz b/tests/data/minimum_Furstenau2025_PMO.json.gz new file mode 100644 index 0000000..b87e2e6 Binary files /dev/null and b/tests/data/minimum_Furstenau2025_PMO.json.gz differ diff --git a/tests/test_pmo_builder/test_metatable_to_pmo.py b/tests/test_pmo_builder/test_metatable_to_pmo.py index 38d59bc..7223cca 100644 --- a/tests/test_pmo_builder/test_metatable_to_pmo.py +++ b/tests/test_pmo_builder/test_metatable_to_pmo.py @@ -1603,13 +1603,18 @@ def test_library_sample_info_table_to_pmo_with_parasite_density(self): parasite_density_method_col="parasite_density_method", ) - self.assertEqual(result[0]["parasite_density_info"][0]["parasite_density"], 10) self.assertEqual( - result[0]["parasite_density_info"][0]["parasite_density_method"], "qPCR" + result[0]["qpcr_parasite_density_info"][0]["parasite_density"], 10 ) - self.assertEqual(result[1]["parasite_density_info"][0]["parasite_density"], 100) self.assertEqual( - result[1]["parasite_density_info"][0]["parasite_density_method"], + result[0]["qpcr_parasite_density_info"][0]["parasite_density_method"], + "qPCR", + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density"], 100 + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density_method"], "microscopy", ) @@ -1638,10 +1643,18 @@ def test_library_sample_info_table_to_pmo_with_parasite_density_multiple(self): parasite_density_method_col=["method1", "method2"], ) - self.assertEqual(result[0]["parasite_density_info"][0]["parasite_density"], 15) - self.assertEqual(result[0]["parasite_density_info"][1]["parasite_density"], 10) - self.assertEqual(result[1]["parasite_density_info"][0]["parasite_density"], 107) - self.assertEqual(result[1]["parasite_density_info"][1]["parasite_density"], 100) + self.assertEqual( + result[0]["qpcr_parasite_density_info"][0]["parasite_density"], 15 + ) + self.assertEqual( + result[0]["qpcr_parasite_density_info"][1]["parasite_density"], 10 + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density"], 107 + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][1]["parasite_density"], 100 + ) def test_library_sample_info_table_to_pmo_with_all_new_fields(self): """Test all new optional fields together""" @@ -1685,7 +1698,9 @@ def test_library_sample_info_table_to_pmo_with_all_new_fields(self): self.assertEqual(result[0]["experiment_accession"], "EXP001") self.assertEqual(result[0]["fastqs_loc"], "/path/to/fastqs1") self.assertEqual(result[0]["run_accession"], "RUN001") - self.assertEqual(result[0]["parasite_density_info"][0]["parasite_density"], 10) + self.assertEqual( + result[0]["qpcr_parasite_density_info"][0]["parasite_density"], 10 + ) self.assertIn("library_prep_plate_info", result[0]) self.assertEqual(result[0]["library_prep_plate_info"]["plate_col"], 1) @@ -1693,7 +1708,9 @@ def test_library_sample_info_table_to_pmo_with_all_new_fields(self): self.assertEqual(result[1]["experiment_accession"], "EXP002") self.assertEqual(result[1]["fastqs_loc"], "/path/to/fastqs2") self.assertEqual(result[1]["run_accession"], "RUN002") - self.assertEqual(result[1]["parasite_density_info"][0]["parasite_density"], 100) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density"], 100 + ) self.assertIn("library_prep_plate_info", result[1]) self.assertEqual(result[1]["library_prep_plate_info"]["plate_col"], 2) diff --git a/tests/test_pmo_builder/test_mhap_table_to_pmo.py b/tests/test_pmo_builder/test_mhap_table_to_pmo.py index fad711b..b50903c 100644 --- a/tests/test_pmo_builder/test_mhap_table_to_pmo.py +++ b/tests/test_pmo_builder/test_mhap_table_to_pmo.py @@ -510,52 +510,49 @@ def test_create_representative_microhaplotype_dict_with_ad_cols(self): mhap_table_ad_cols = self.small_mhap_table.copy() mhap_table_ad_cols["adcol1"] = "this" mhap_table_ad_cols["adcol2"] = "that" - mhap_table_ad_cols["cig"] = "cigs" rep_dict_with_ad_cols = copy.deepcopy(self.small_representative_dict) - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][0]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][0]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][0][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][1]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][1]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][1][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][2]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][2]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][2][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][0]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][0]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][0][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][1]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][1]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][1][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][0]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][0]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][0][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][1]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][1]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][1][ - "pseudo_cigar" - ] = "cigs" + for target in rep_dict_with_ad_cols["targets"]: + for mhap in target["microhaplotypes"]: + mhap["adcol1"] = "this" + mhap["adcol2"] = "that" actual = create_representative_microhaplotype_dict( mhap_table_ad_cols, additional_representative_mhap_cols=["adcol1", "adcol2"], - pseudocigar_col="cig", ) self.assertEqual(actual, rep_dict_with_ad_cols) + def test_create_representative_microhaplotype_dict_with_pseudocigar(self): + tbl = self.small_mhap_table.copy() + tbl["chrom"] = "chr1" + tbl["start"] = 100 + tbl["end"] = 150 + tbl["cig"] = "8M" + actual = create_representative_microhaplotype_dict( + tbl, + chrom_col="chrom", + start_col="start", + end_col="end", + pseudocigar_col="cig", + pseudocigar_start_col="start", + pseudocigar_end_col="end", + ) + first = actual["targets"][0]["microhaplotypes"][0] + # pseudocigar is a Pseudocigar object with a GenomicLocation ref_loc; + # the chromosome is reused from chrom_col and genome_id defaults to 0 + self.assertEqual(first["pseudocigar"]["pseudocigar_seq"], "8M") + self.assertEqual( + first["pseudocigar"]["ref_loc"], + {"genome_id": 0, "chrom": "chr1", "start": 100, "end": 150}, + ) + + def test_pseudocigar_without_ref_loc_columns_raises(self): + tbl = self.small_mhap_table.copy() + tbl["cig"] = "8M" + with self.assertRaises(ValueError): + create_representative_microhaplotype_dict(tbl, pseudocigar_col="cig") + @patch( "pmotools.pmo_builder.mhap_table_to_pmo.create_representative_microhaplotype_dict" ) diff --git a/tests/test_pmo_builder/test_panel_information_to_pmo.py b/tests/test_pmo_builder/test_panel_information_to_pmo.py index 29158a3..bf74cf3 100644 --- a/tests/test_pmo_builder/test_panel_information_to_pmo.py +++ b/tests/test_pmo_builder/test_panel_information_to_pmo.py @@ -416,8 +416,8 @@ def test_build_target_info_dict_full_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": { @@ -443,8 +443,8 @@ def test_build_target_info_dict_full_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": { @@ -470,8 +470,8 @@ def test_build_target_info_dict_full_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": { @@ -526,8 +526,8 @@ def test_build_target_info_dict_missing_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": {"seq": "GTT"}, diff --git a/tests/test_pmo_builder/test_schema_validation_integration.py b/tests/test_pmo_builder/test_schema_validation_integration.py index 766bca5..5f88b9b 100644 --- a/tests/test_pmo_builder/test_schema_validation_integration.py +++ b/tests/test_pmo_builder/test_schema_validation_integration.py @@ -191,6 +191,8 @@ def test_full_1_0_0_toy_pmo_validates_against_schema(): umis_col="umis", microhaplotype_name_col="microhap_name", pseudocigar_col="pseudocigar", + pseudocigar_start_col="start", + pseudocigar_end_col="end", quality_col="quality", masking_seq_start_col="mask_start", masking_seq_segment_size_col="mask_segment", @@ -467,6 +469,9 @@ def test_slimer_1_1_0_toy_pmo_validates_against_schema(): "umis": [10], "microhap_name": ["mh1"], "pseudocigar": ["8M"], + "pseudocigar_chrom": ["chr1"], + "pseudocigar_start": [120], + "pseudocigar_end": [127], "quality": ["ABCD"], "mask_start": ["1"], "mask_segment": ["2"], @@ -480,6 +485,9 @@ def test_slimer_1_1_0_toy_pmo_validates_against_schema(): umis_col="umis", microhaplotype_name_col="microhap_name", pseudocigar_col="pseudocigar", + pseudocigar_chrom_col="pseudocigar_chrom", + pseudocigar_start_col="pseudocigar_start", + pseudocigar_end_col="pseudocigar_end", quality_col="quality", masking_seq_start_col="mask_start", masking_seq_segment_size_col="mask_segment", diff --git a/tests/test_pmo_engine/test_pmo_exporter.py b/tests/test_pmo_engine/test_pmo_exporter.py index 1984bfa..44e2389 100755 --- a/tests/test_pmo_engine/test_pmo_exporter.py +++ b/tests/test_pmo_engine/test_pmo_exporter.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import copy import gzip import hashlib import os @@ -38,6 +39,15 @@ def setUp(self): "rt", ) as f: self.minimum_fields_v1_1_0_pmo_data = json.load(f) + # a real, schema-valid v1.1.0 PMO with only the required top-level sections + with gzip.open( + os.path.join( + os.path.dirname(self.working_dir), + "data/minimum_Furstenau2025_PMO.json.gz", + ), + "rt", + ) as f: + self.minimum_v1_1_0_pmo_data = json.load(f) def tearDown(self): self.test_dir.cleanup() @@ -105,6 +115,54 @@ def test_extract_panels_insert_bed_loc(self): PMOExporter.write_bed_locs(all_target_inserts, output_fnp) self.assertEqual("52b1f79a3a89f8265573fa54b5a7ce57", md5sum_of_fnp(output_fnp)) + def test_extract_panels_insert_bed_loc_covers_all_reactions(self): + # regression: previously the function returned early inside the reaction + # loop, so only the first reaction of the first panel was returned. + import copy + + pmo = copy.deepcopy(self.combined_pmo_data) + panel = pmo["panel_info"][0] + targets = panel["reactions"][0]["panel_targets"] + half = len(targets) // 2 + panel["reactions"] = [ + {"reaction_name": "pool1", "panel_targets": targets[:half]}, + {"reaction_name": "pool2", "panel_targets": targets[half:]}, + ] + bed_locs = PMOExporter.extract_panels_insert_bed_loc(pmo, sort_output=False) + # every target across BOTH reactions is present + self.assertEqual(len(bed_locs), len(targets)) + self.assertEqual( + {b.name for b in bed_locs}, + {pmo["target_info"][t]["target_name"] for t in targets}, + ) + + def test_write_bed_locs_header_on_own_line(self): + # regression: header was written without a trailing newline, gluing the + # first data row onto it. + bed_locs = PMOExporter.extract_targets_insert_bed_loc( + self.combined_pmo_data, sort_output=True + ) + out_fnp = os.path.join(self.test_dir.name, "with_header.bed") + PMOExporter.write_bed_locs(bed_locs, out_fnp, add_header=True) + with open(out_fnp) as f: + lines = f.read().splitlines() + self.assertEqual( + lines[0], + "\t".join( + [ + "#chrom", + "start", + "end", + "name", + "score", + "strand", + "ref_seq", + "extra_info", + ] + ), + ) + self.assertEqual(len(lines), len(bed_locs) + 1) + def test_extract_alleles_per_sample_table(self): allele_data = PMOExporter.extract_alleles_per_sample_table( self.combined_pmo_data, @@ -245,6 +303,28 @@ def test_export_library_sample_meta_table(self): md5sum_of_fnp(os.path.join(self.test_dir.name, "library_sample_table.csv")), ) + def test_export_meta_tables_without_optional_cross_ref_sections(self): + # project_info and sequencing_info are optional as of v1.1.0; the specimen and + # library_sample meta-table exporters must not raise a raw KeyError when a + # referencing id is present but the referenced optional section is absent + self.assertNotIn("project_info", self.minimum_v1_1_0_pmo_data) + self.assertNotIn("sequencing_info", self.minimum_v1_1_0_pmo_data) + # a clean minimal PMO (no dangling ids) should export fine + PMOExporter.export_specimen_meta_table(self.minimum_v1_1_0_pmo_data) + PMOExporter.export_library_sample_meta_table(self.minimum_v1_1_0_pmo_data) + + # inject the referencing ids without the optional sections (referential dangling) + # the raw id is kept rather than resolving an unavailable name + dangling = copy.deepcopy(self.minimum_v1_1_0_pmo_data) + dangling["specimen_info"][0]["project_id"] = 0 + dangling["library_sample_info"][0]["sequencing_info_id"] = 0 + spec_table = PMOExporter.export_specimen_meta_table(dangling) + self.assertIn("project_id", spec_table.columns) + self.assertNotIn("project_name", spec_table.columns) + library_table = PMOExporter.export_library_sample_meta_table(dangling) + self.assertIn("sequencing_info_id", library_table.columns) + self.assertNotIn("sequencing_info_name", library_table.columns) + def test_export_sequencing_info_meta_table(self): sequencing_info_table = PMOExporter.export_sequencing_info_meta_table( self.small_example_pmo_data diff --git a/tests/test_pmo_engine/test_pmo_processor.py b/tests/test_pmo_engine/test_pmo_processor.py index 2c4a41b..729f64d 100755 --- a/tests/test_pmo_engine/test_pmo_processor.py +++ b/tests/test_pmo_engine/test_pmo_processor.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import copy +import gzip import os import tempfile import unittest @@ -36,6 +38,16 @@ def setUp(self): ) ) as f: self.minimum_pmo_data = json.load(f) + # a real, schema-valid v1.1.0 PMO that contains only the required top-level + # sections (no sequencing_info/project_info/targeted_genomes/bioinformatics_*/ + # read_counts_by_stage) to guard against unguarded optional-section access + with gzip.open( + os.path.join( + os.path.dirname(self.working_dir), + "data/minimum_Furstenau2025_PMO.json.gz", + ) + ) as f: + self.minimum_v1_1_0_pmo_data = json.load(f) def tearDown(self): self.test_dir.cleanup() @@ -317,10 +329,52 @@ def test_extract_from_pmo_with_read_filter(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_filtered, f) - self.assertEqual("879b1a0c62a77a8fcc910a21d8ec9de6", md5sum_of_fnp(output_fnp)) + self.assertEqual("59e24e7efd4cd7822acde70828ad6119", md5sum_of_fnp(output_fnp)) checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_filtered) + def test_extract_from_pmo_with_read_filter_missing_optional_sections(self): + # the v1.1.0 schema relaxed the required fields, so a PMO may omit these + # optional sections; the extracted PMO must not copy them in as null/empty keys + optional_sections = [ + "sequencing_info", + "project_info", + "targeted_genomes", + "bioinformatics_methods_info", + "bioinformatics_run_info", + "read_counts_by_stage", + ] + stripped_pmo_data = copy.deepcopy(self.combined_pmo_data) + for section in optional_sections: + stripped_pmo_data.pop(section, None) + pmo_data_filtered = PMOProcessor.extract_from_pmo_with_read_filter( + stripped_pmo_data, 1000 + ) + for section in optional_sections: + self.assertNotIn(section, pmo_data_filtered) + # the relaxed required fields are a v1.1.0 change, so validate against v1.1.0 + checker = PMOChecker( + load_schema("portable_microhaplotype_object_v1.1.0.schema.json") + ) + checker.validate_pmo_json(pmo_data_filtered) + + def test_bioinformatics_run_name_lookups_without_run_info(self): + # bioinformatics_run_info is optional as of v1.1.0; the run-name lookups + # must not raise a raw KeyError on a valid PMO that omits it + self.assertNotIn("bioinformatics_run_info", self.minimum_v1_1_0_pmo_data) + self.assertEqual( + {}, + PMOProcessor.get_index_key_of_bioinformatics_run_names( + self.minimum_v1_1_0_pmo_data + ), + ) + self.assertEqual( + [], + PMOProcessor.get_sorted_bioinformatics_run_names( + self.minimum_v1_1_0_pmo_data + ), + ) + def test_filter_pmo_by_target_ids(self): pmo_data_select_targets = PMOProcessor.filter_pmo_by_target_ids( self.combined_pmo_data, {1, 10, 11, 55} @@ -358,7 +412,7 @@ def test_filter_pmo_by_library_sample_ids(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_targets, f) - self.assertEqual("4a2c7b9d84322cc43adf2b9f0c518244", md5sum_of_fnp(output_fnp)) + self.assertEqual("36d5cb8a79962707374fae86ba1a6f90", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_targets) @@ -374,7 +428,7 @@ def test_filter_pmo_by_library_sample_names(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_library_sample_names, f) - self.assertEqual("ffdaeb6b57ed9a5c51e56cfaa4621796", md5sum_of_fnp(output_fnp)) + self.assertEqual("6bd4e6367a62f1eb3c73f481a5ae9013", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_library_sample_names) @@ -388,7 +442,7 @@ def test_filter_pmo_by_specimen_ids(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_targets, f) - self.assertEqual("f3dd2e0aee9afa82336ff48475a3c69c", md5sum_of_fnp(output_fnp)) + self.assertEqual("664de0f14180de852f3f5d8942454004", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_targets) @@ -402,7 +456,7 @@ def test_filter_pmo_by_specimen_names(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_targets, f) - self.assertEqual("f3dd2e0aee9afa82336ff48475a3c69c", md5sum_of_fnp(output_fnp)) + self.assertEqual("664de0f14180de852f3f5d8942454004", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_targets) @@ -420,7 +474,7 @@ def test_extract_from_pmo_samples_with_meta_groupings(self): with open(output_fnp, "w") as f: json.dump(pmo_data_select_meta, f) - self.assertEqual("f0f67e399885824a7aa318ead8b3d09c", md5sum_of_fnp(output_fnp)) + self.assertEqual("9deb0bccc0017593697190d45ae6c0a4", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_meta) diff --git a/tests/test_pmo_engine/test_pmo_reader.py b/tests/test_pmo_engine/test_pmo_reader.py index b612ae2..0075a48 100755 --- a/tests/test_pmo_engine/test_pmo_reader.py +++ b/tests/test_pmo_engine/test_pmo_reader.py @@ -165,6 +165,73 @@ def test_combine_multiple_pmos_fail_dup_library_sample_names(self): ) self.assertRaises(Exception, PMOReader.combine_multiple_pmos, pmo_data_list_2) + def test_combine_multiple_pmos_remaps_new_rep_target_id(self): + # a representative-microhaplotype target newly added from a + # non-first PMO must have its target_id remapped to the combined + # target_info (previously it kept its local target_id), this test + # ensures this is tested as it was previously untested (2026-06-29) + import pandas as pd + from pmotools.pmo_builder.mhap_table_to_pmo import mhap_table_to_pmo + from pmotools.pmo_builder.panel_information_to_pmo import ( + panel_info_table_to_pmo, + ) + from pmotools.pmo_builder.merge_to_pmo import merge_to_pmo + + def build(libs, targets, seqs, panel_name, panel_targets): + mhap_info = mhap_table_to_pmo( + pd.DataFrame( + { + "library_sample_name": libs, + "target_name": targets, + "seq": seqs, + "reads": [10] * len(libs), + } + ) + ) + panel_info = panel_info_table_to_pmo( + pd.DataFrame( + { + "target_name": panel_targets, + "fwd_primer": [ + "A" * (i + 4) for i in range(len(panel_targets)) + ], + "rev_primer": [ + "C" * (i + 4) for i in range(len(panel_targets)) + ], + } + ), + panel_name, + ) + return merge_to_pmo(mhap_info=mhap_info, panel_target_info=panel_info) + + pmo_a = build( + ["S1", "S2"], ["t1", "t2"], ["AAA", "GGG"], "panelA", ["t1", "t2"] + ) + # pmo_b introduces a brand-new target t3 + pmo_b = build( + ["S3", "S4"], ["t1", "t3"], ["AAA", "CCC"], "panelB", ["t1", "t3"] + ) + + combined = PMOReader.combine_multiple_pmos([pmo_a, pmo_b]) + self.assertEqual( + [t["target_name"] for t in combined["target_info"]], + ["t1", "t2", "t3"], + ) + # every rep target's target_id resolves to the right combined target + for rep in combined["representative_microhaplotypes"]["targets"]: + self.assertLess(rep["target_id"], len(combined["target_info"])) + t3_reps = [ + rep + for rep in combined["representative_microhaplotypes"]["targets"] + if combined["target_info"][rep["target_id"]]["target_name"] == "t3" + ] + self.assertEqual(len(t3_reps), 1) + # validate against schema (minimal builder PMO targets v1.1.0) + checker = PMOChecker( + load_schema("portable_microhaplotype_object_v1.1.0.schema.json") + ) + checker.validate_pmo_json(combined) + def test_combine_multiple_pmos_fail_for_combine_only_one_file(self): # will fail for only having 1 PMO pmo_data_list_2 = PMOReader.read_in_pmos(