From 029dedd2c8664b614ba64599879a789c0988f3d8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:34:09 +0100 Subject: [PATCH 01/86] refactor(ancestry): og archaic and og ancient in Rust, with a chain liftover og archaic table|strict|genotypes|report and og ancient select|lift|person-geno|align|report port bin/archaic_report.py and bin/ancient_report.py. og-intervals gains chain, the UCSC chain reader with a single-position lift. Bead: pgp-bs9u.21 --- rust/Cargo.lock | 1 + rust/crates/og-ancestry/Cargo.toml | 3 +- rust/crates/og-ancestry/src/ancient.rs | 1569 +++++++++++++++++++++ rust/crates/og-ancestry/src/archaic.rs | 1728 ++++++++++++++++++++++++ rust/crates/og-ancestry/src/lib.rs | 2 + rust/crates/og-intervals/src/chain.rs | 376 ++++++ rust/crates/og-intervals/src/lib.rs | 3 + rust/og/src/main.rs | 12 + 8 files changed, 3693 insertions(+), 1 deletion(-) create mode 100644 rust/crates/og-ancestry/src/ancient.rs create mode 100644 rust/crates/og-ancestry/src/archaic.rs create mode 100644 rust/crates/og-intervals/src/chain.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 86841ba6c..c25f174d4 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -601,6 +601,7 @@ name = "og-ancestry" version = "0.1.0" dependencies = [ "flate2", + "og-intervals", "og-io", "og-tsv", "pycompat", diff --git a/rust/crates/og-ancestry/Cargo.toml b/rust/crates/og-ancestry/Cargo.toml index b27b4bbfa..4406d01ba 100644 --- a/rust/crates/og-ancestry/Cargo.toml +++ b/rust/crates/og-ancestry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-ancestry" -description = "Step 26b, fine-grained ancestry: og ancestry-fine panel-map, check-map, order, sample-bed, chroms and report" +description = "Ancestry steps: og ancestry-fine (26b), og archaic (43) and og ancient (44)" version.workspace = true edition.workspace = true license.workspace = true @@ -9,6 +9,7 @@ publish.workspace = true [dependencies] pycompat.workspace = true og-io.workspace = true +og-intervals.workspace = true og-tsv.workspace = true flate2.workspace = true diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs new file mode 100644 index 000000000..612a71212 --- /dev/null +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -0,0 +1,1569 @@ +//! Step 44, ancient similarity (`docs/44-ancient.md`): where a genome falls +//! among ancient genomes on a PCA of present-day West Eurasians (smartpca, +//! EIGENSOFT), with the ancient samples of the Allen Ancient DNA Resource +//! (AADR) projected onto the same axes, as `og ancient `. +//! +//! - `select` (setup): reads the AADR `.anno` and `.ind` files and chooses +//! the people: the present-day West Eurasians of [`MODERN_GROUPS`] build +//! the axes, the ancient West Eurasians with enough sites are projected. +//! - `lift` (setup): moves the 1240K sites of the AADR `.snp` file from +//! GRCh37 to GRCh38 with UCSC's chain file ([`og_intervals::chain`]) and +//! keeps a site only when one of its two alleles is the GRCh38 base. +//! - `person-geno` (step 44): the person's genotype at each site, appended +//! as one more column to the present-day panel's EIGENSTRAT `.geno`, +//! streamed row by row. +//! - `align` (setup, step 44): checks that the axes of two smartpca runs on +//! the same panel agree, up to the sign of each. +//! - `report` (step 44): the nearest ancient populations, the summary and +//! the plot. +//! +//! Each command writes what `bin/ancient_report.py` wrote (checked by +//! `tests/parity`). Deliberate changes: every output is written under a +//! temporary name and renamed into place (the Python wrote the population +//! list, `ancients.tsv`, the stats and the text report in place); a gzip +//! input is told by its first bytes, not its name; a `.fai` that does not +//! match its FASTA stops `lift`; and malformed input is an `ERROR:` line +//! and exit 1, never a traceback. + +use super::{ArgError, Args, Fail, Pending, Result, Spec, cannot_read, int, repr, split_ws}; +use og_intervals::chain::Chain; +use og_intervals::fasta::Indexed; +use og_io::{Lines, Utf8}; +use pycompat::fmt::{fixed, int_grouped, int_str, ljust, rjust, round_int}; +use pycompat::text::{html_escape, strip, upper}; +use std::collections::{BTreeMap, HashMap}; +use std::io::{BufRead, Write}; +use std::path::{Path, PathBuf}; + +/// Present-day groups of the AADR 1240K file (Group ID, as written in the +/// `.anno` file of v66.p1) that build the axes: Europe, the Caucasus and +/// the Near East. North African groups and groups with recent East +/// Eurasian ancestry are left out, so the axes describe West Eurasian +/// variation only. +pub const MODERN_GROUPS: [&str; 41] = [ + "Abkhasian", + "Adygei", + "Albanian", + "Armenian", + "Basque", + "BedouinA", + "BedouinB", + "Bulgarian", + "CEU", + "Chechen", + "Czech", + "Druze", + "English", + "Estonian", + "FIN", + "Finnish", + "French", + "GBR", + "Georgian", + "Greek", + "Greek_Crete", + "Hungarian", + "IBS", + "Icelandic", + "Iranian", + "Italian_North", + "Jew_Iraqi", + "Jordanian", + "Lezgin", + "Norwegian", + "Orcadian", + "Ossetian", + "Palestinian", + "Polish", + "Russian", + "Sardinian", + "Scottish", + "Spanish", + "Spanish_North", + "TSI", + "Turkish", +]; +const MODERN_DATA_TYPE: &str = "Shotgun.diploid"; +/// Ancient samples projected: dated, passing the AADR's own assessment, +/// with this many of the 1240K autosomal sites covered, inside West Eurasia. +const MIN_ANCIENT_SNPS: i64 = 100_000; +const LAT: (f64, f64) = (25.0, 75.0); +const LON: (f64, f64) = (-25.0, 75.0); +const PASSING: [&str; 3] = ["PASS", "PROVISIONAL_PASS", "MERGE_PASS"]; +/// The label of a projected ancient person in the `.ind` and `.evec` files. +pub const ANCIENT_LABEL: &str = "Ancient"; +/// The label of the person in the run's `.evec`. +pub const PERSON_LABEL: &str = "Person"; +/// PCs the distance uses, each weighted by the square root of its eigenvalue. +const DISTANCE_PCS: i64 = 4; +const TOP: usize = 20; +/// Two runs on the same panel give the same axes; a PC whose present-day +/// coordinates correlate less than this between the two is a mismatch. +const MIN_AXIS_CORRELATION: f64 = 0.999; +const CAVEAT: &str = "Closeness on a PCA is genetic similarity, not descent: a genome near an ancient group \ +shares ancestry with it, it does not descend from those people."; +const QPADM_NOTE: &str = "qpAdm models of ancestry proportions are not part of this step."; + +/// The `.anno` columns read, matched by the start of the header cell (the +/// AADR writes long explanations after the name). +const ANNO_COLUMNS: [(Col, &str); 13] = [ + (Col::Id, "Genetic ID"), + (Col::Individual, "Individual ID"), + (Col::DateBp, "Date mean in BP"), + (Col::DateSd, "Date standard deviation in BP"), + (Col::FullDate, "Full Date"), + (Col::Group, "Group ID"), + (Col::Locality, "Locality"), + (Col::Country, "Political Entity"), + (Col::Lat, "Latitude"), + (Col::Lon, "Longitude"), + (Col::DataType, "Data type"), + ( + Col::Snps, + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", + ), + (Col::Assessment, "ASSESSMENT"), +]; + +/// A column of the `.anno` file this tool reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Col { + Id, + Individual, + DateBp, + DateSd, + FullDate, + Group, + Locality, + Country, + Lat, + Lon, + DataType, + Snps, + Assessment, +} + +/// One `.anno` row: the 13 cells read, stripped, in [`ANNO_COLUMNS`] order. +#[derive(Debug, Clone)] +struct Anno([String; 13]); + +impl Anno { + fn get(&self, c: Col) -> &str { + &self.0[c as usize] + } +} + +/// A text file read line by line as Python's text mode reads it. +struct Text { + path: PathBuf, + lines: Lines>, + line: String, +} + +impl Text { + fn open(path: &str, utf8: Utf8) -> Result { + let path = PathBuf::from(path); + let input = og_io::open(&path).map_err(|e| cannot_read(&path, &e))?; + Ok(Text { + path, + lines: Lines::new(input, utf8), + line: String::new(), + }) + } + + /// The next line with its `\n`, or `None` at the end. + fn raw(&mut self) -> Result> { + let more = self + .lines + .read_line(&mut self.line) + .map_err(|e| cannot_read(&self.path, &e))?; + Ok(more.then_some(self.line.as_str())) + } + + /// The next line without its `\n`. + fn next(&mut self) -> Result> { + Ok(self.raw()?.map(|l| l.strip_suffix('\n').unwrap_or(l))) + } +} + +/// Python's `float(s)` of a finite value, or `None`. +fn to_float(s: &str) -> Option { + pycompat::num::parse_float(s).ok().filter(|v| v.is_finite()) +} + +/// Python's `float(s)`, or the input error that names the file. +fn need_float(s: &str, path: &Path) -> Result { + pycompat::num::parse_float(s) + .map_err(|_| Fail(format!("{}: {} is not a number", path.display(), repr(s)))) +} + +/// Python's `f >= i` for a float and an int: exact, whatever their sizes. +fn float_ge_int(f: f64, i: i64) -> bool { + let fi = i as f64; + if f != fi { + return f > fi; + } + // f equals the float nearest i, so it is a whole number. + (f as i128) >= i128::from(i) +} + +/// `" ".join(str(s).replace("\t", " ").split()) or "."`. +fn clean(s: &str) -> String { + let out = split_ws(s).collect::>().join(" "); + if out.is_empty() { ".".to_owned() } else { out } +} + +// --- select ----------------------------------------------------------------------- + +/// The index of each read column in the `.anno` header. +fn anno_columns(header: &[&str]) -> Result<[usize; 13]> { + let cells: Vec<&str> = header + .iter() + .map(|c| strip(strip(strip(c, None), Some("\"")), None)) + .collect(); + let mut out = [0usize; 13]; + for (k, (col, name)) in ANNO_COLUMNS.iter().enumerate() { + // ASSESSMENT must not match ASSESSMENT WARNINGS: the exact cell first. + let exact = cells.iter().position(|c| c == name); + let hit = if *col == Col::Assessment { + exact + } else { + exact.or_else(|| cells.iter().position(|c| c.starts_with(name))) + }; + out[k] = hit.ok_or_else(|| Fail(format!("the .anno file has no column '{name}'")))?; + } + Ok(out) +} + +fn passing(assessment: &str) -> bool { + let a = upper(strip(assessment, None)); + PASSING.iter().any(|p| a.starts_with(p)) +} + +/// What [`choose`] keeps of one genetic id. +#[derive(Debug, Clone)] +enum Chosen { + /// A present-day reference person, with their group. + Modern(String), + /// An ancient person to project, with their row. + Ancient(Box), +} + +/// The present-day reference people and the ancient people to project, +/// by genetic id, one genetic id per individual (the one with the most +/// sites). Rows are streamed from the `.anno` file. +fn choose(path: &str, min_snps: i64) -> Result> { + let mut t = Text::open(path, Utf8::Replace)?; + let Some(head) = t.next()? else { + return Err(Fail(format!("{path} is empty"))); + }; + let header: Vec<&str> = head.split('\t').collect(); + let col = anno_columns(&header)?; + let widest = *col.iter().max().expect("13 columns"); + // (kind is modern, individual) -> (sites, row), in first-seen order. + let mut index: HashMap<(bool, String), usize> = HashMap::new(); + let mut best: Vec<(f64, Anno, bool)> = Vec::new(); + while let Some(line) = t.next()? { + let cells: Vec<&str> = line.split('\t').collect(); + if line.is_empty() || cells.len() <= widest { + continue; + } + let r = Anno(col.map(|i| strip(cells[i], None).to_owned())); + if !passing(r.get(Col::Assessment)) { + continue; + } + let bp = to_float(r.get(Col::DateBp)); + let snps = to_float(r.get(Col::Snps)).unwrap_or(0.0); + let modern = if bp == Some(0.0) + && MODERN_GROUPS.contains(&r.get(Col::Group)) + && r.get(Col::DataType) == MODERN_DATA_TYPE + { + true + } else if bp.is_some_and(|b| b > 0.0) && float_ge_int(snps, min_snps) { + let (Some(lat), Some(lon)) = (to_float(r.get(Col::Lat)), to_float(r.get(Col::Lon))) + else { + continue; + }; + if !(LAT.0 <= lat && lat <= LAT.1 && LON.0 <= lon && lon <= LON.1) { + continue; + } + false + } else { + continue; + }; + let who = if r.get(Col::Individual).is_empty() { + r.get(Col::Id) + } else { + r.get(Col::Individual) + }; + let key = (modern, who.to_owned()); + match index.get(&key) { + None => { + index.insert(key, best.len()); + best.push((snps, r, modern)); + } + Some(&i) if snps > best[i].0 => best[i] = (snps, r, modern), + Some(_) => {} + } + } + let mut out = HashMap::new(); + for (_, r, modern) in best { + let id = r.get(Col::Id).to_owned(); + let c = if modern { + Chosen::Modern(r.get(Col::Group).to_owned()) + } else { + Chosen::Ancient(Box::new(r)) + }; + out.insert(id, c); + } + Ok(out) +} + +const SELECT: Spec = Spec { + verb: "select", + usage: "usage: og ancient select --anno ANNO --ind IND --out-ind OUT_IND --poplist POPLIST --ancients ANCIENTS [--min-snps MIN_SNPS]\n", + options: &[ + "--anno", + "--ind", + "--out-ind", + "--poplist", + "--ancients", + "--min-snps", + ], + required: &["--anno", "--ind", "--out-ind", "--poplist", "--ancients"], +}; + +fn cmd_select(a: &Args, out: &mut dyn Write) -> Result<()> { + let min_snps = a + .int("--min-snps", MIN_ANCIENT_SNPS) + .unwrap_or(MIN_ANCIENT_SNPS); + let chosen = choose(a.req("--anno"), min_snps)?; + let (mut n_mod, mut n_anc) = (0u64, 0u64); + let mut groups: std::collections::BTreeSet = Default::default(); + let mut ind = String::new(); + let mut t = Text::open(a.req("--ind"), Utf8::Strict)?; + while let Some(line) = t.next()? { + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 3 { + continue; + } + let label = match chosen.get(c[0]) { + Some(Chosen::Modern(g)) => { + n_mod += 1; + groups.insert(g.clone()); + g.as_str() + } + Some(Chosen::Ancient(_)) => { + n_anc += 1; + ANCIENT_LABEL + } + None => "Ignore", + }; + ind += &format!("{}\t{}\t{label}\n", c[0], c[1]); + } + let missing: Vec<&str> = { + let mut m: Vec<&str> = MODERN_GROUPS + .iter() + .copied() + .filter(|g| !groups.contains(*g)) + .collect(); + m.sort_unstable(); + m + }; + if n_mod < 50 || n_anc < 1 { + return Err(Fail(format!( + "the .ind file matches {n_mod} present-day reference people and {n_anc} ancient ones; \ + is it the .ind file of the same release as the .anno file?" + ))); + } + let mut files = Pending::new(); + files.put(PathBuf::from(a.req("--out-ind")), ind.as_bytes())?; + let pops: String = groups.iter().map(|g| format!("{g}\n")).collect(); + files.put(PathBuf::from(a.req("--poplist")), pops.as_bytes())?; + let mut anc = + String::from("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"); + let sorted: BTreeMap<&String, &Chosen> = chosen.iter().collect(); + for c in sorted.values() { + let Chosen::Ancient(r) = c else { continue }; + let cells: Vec = [ + Col::Id, + Col::Group, + Col::DateBp, + Col::DateSd, + Col::FullDate, + Col::Locality, + Col::Country, + Col::Lat, + Col::Lon, + Col::Snps, + ] + .iter() + .map(|k| clean(r.get(*k))) + .collect(); + anc += &cells.join("\t"); + anc.push('\n'); + } + files.put(PathBuf::from(a.req("--ancients")), anc.as_bytes())?; + files.commit_all()?; + let suffix = if !missing.is_empty() && missing.len() <= 10 { + format!(" (not in this file: {})", missing.join(", ")) + } else if !missing.is_empty() { + format!( + " ({} of the {} groups are not in this file)", + missing.len(), + MODERN_GROUPS.len() + ) + } else { + String::new() + }; + let _ = writeln!( + out, + "Present-day reference people: {n_mod} in {} groups{suffix}", + groups.len() + ); + let _ = writeln!( + out, + "Ancient people projected: {n_anc} (at least {} sites, West Eurasia)", + int_grouped(i128::from(min_snps)) + ); + Ok(()) +} + +// --- lift ------------------------------------------------------------------------- + +/// A base's complement, for the four bases. +fn complement(b: &str) -> Option<&'static str> { + match b { + "A" => Some("T"), + "C" => Some("G"), + "G" => Some("C"), + "T" => Some("A"), + _ => None, + } +} + +/// The chromosome column of an EIGENSOFT `.snp` line: 1 to 22, or `None`. +fn eigen_chrom(c: &str) -> Option { + int(c).filter(|n| (1..=22).contains(n)) +} + +const LIFT: Spec = Spec { + verb: "lift", + usage: "usage: og ancient lift --snp SNP --chain CHAIN --fasta FASTA --out OUT\n", + options: &["--snp", "--chain", "--fasta", "--out"], + required: &["--snp", "--chain", "--fasta", "--out"], +}; + +fn cmd_lift(a: &Args, out: &mut dyn Write) -> Result<()> { + let chain_path = a.req("--chain"); + let input = og_io::open(chain_path).map_err(|e| cannot_read(Path::new(chain_path), &e))?; + let chain = Chain::read(input).map_err(|e| Fail(format!("{chain_path}: {e}")))?; + let fasta_path = a.req("--fasta"); + let mut fasta = Indexed::open(fasta_path).map_err(|e| Fail(format!("{fasta_path}: {e}")))?; + let mut kept: Vec<(i64, i64, String, &'static str, &'static str)> = Vec::new(); + let mut dropped: BTreeMap<&str, u64> = BTreeMap::new(); + let mut n: u64 = 0; + let snp_path = a.req("--snp"); + let mut t = Text::open(snp_path, Utf8::Replace)?; + let mut lineno = 0u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 6 { + continue; + } + n += 1; + let Some(chrom) = eigen_chrom(c[1]) else { + *dropped.entry("not an autosome").or_default() += 1; + continue; + }; + let (u1, u2) = (upper(c[4]), upper(c[5])); + let (Some(k1), Some(k2)) = (complement(&u1), complement(&u2)) else { + *dropped + .entry("alleles not two different bases") + .or_default() += 1; + continue; + }; + if u1 == u2 { + *dropped + .entry("alleles not two different bases") + .or_default() += 1; + continue; + } + let pos = pycompat::num::parse_float(c[3]) + .ok() + .filter(|p| p.is_finite()) + .map(f64::trunc) + .filter(|p| p.abs() < 9.0e18) + .ok_or_else(|| { + Fail(format!( + "{snp_path}: line {lineno}: position {} is not a number", + repr(c[3]) + )) + })? as i64; + let name = format!("chr{chrom}"); + let Some(hit) = chain.lift(&name, pos) else { + *dropped.entry("no single place on GRCh38").or_default() += 1; + continue; + }; + if hit.chrom != name { + *dropped.entry("moves to another chromosome").or_default() += 1; + continue; + } + let (a1, a2) = if hit.strand == "-" { + (k1, k2) + } else { + ( + complement(k1).expect("a base"), + complement(k2).expect("a base"), + ) + }; + let q_pos = hit.pos; + let base = if q_pos >= 1 { + fasta + .base(hit.chrom, (q_pos - 1) as u64) + .map_err(|e| Fail(format!("{fasta_path}: {e}")))? + } else { + None + }; + let base = base.map(|b| b.to_ascii_uppercase()); + if !base.is_some_and(|b| b == a1.as_bytes()[0] || b == a2.as_bytes()[0]) { + *dropped + .entry("neither allele is the GRCh38 base") + .or_default() += 1; + continue; + } + kept.push((chrom, q_pos, c[0].to_owned(), a1, a2)); + } + kept.sort_unstable(); + let mut text = String::from("snp\tchrom\tpos\tallele1\tallele2\n"); + for (chrom, pos, sid, a1, a2) in &kept { + text += &format!("{sid}\tchr{chrom}\t{pos}\t{a1}\t{a2}\n"); + } + let mut files = Pending::new(); + files.put(PathBuf::from(a.req("--out")), text.as_bytes())?; + files.commit_all()?; + let _ = writeln!( + out, + "Sites of the .snp file: {}; on GRCh38: {}", + int_grouped(i128::from(n)), + int_grouped(kept.len() as i128) + ); + for (why, k) in &dropped { + let _ = writeln!(out, " dropped, {why}: {}", int_grouped(i128::from(*k))); + } + if kept.is_empty() { + return Err(Fail( + "no site could be placed on GRCh38: check the chain file and the reference".into(), + )); + } + Ok(()) +} + +// --- person-geno ------------------------------------------------------------------ + +/// Copies of allele1 in a bcftools `%TGT` genotype, or `None` (missing, a +/// haploid or half call, or an allele that is neither of the two). +fn dosage(gt: &str, a1: &str, a2: &str) -> Option { + let mut parts = gt.split(['/', '|']); + let (Some(x), Some(y), None) = (parts.next(), parts.next(), parts.next()) else { + return None; + }; + let mut d = 0; + for al in [x, y] { + if al == a1 { + d += 1; + } else if al != a2 { + return None; + } + } + Some(d) +} + +/// The panel's GRCh38 sites: per (chromosome, position), the sites there +/// (id, allele1, allele2), packed in one array. +struct Sites { + chroms: HashMap, + at: HashMap<(u32, i64), Vec>, + sites: Vec<(Box, Box, Box)>, +} + +fn read_sites(path: &str) -> Result { + let mut s = Sites { + chroms: HashMap::new(), + at: HashMap::new(), + sites: Vec::new(), + }; + let mut t = Text::open(path, Utf8::Strict)?; + let _ = t.next()?; + let mut lineno = 1u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + continue; + } + let pos = int(c[2]).ok_or_else(|| { + Fail(format!( + "{path}: line {lineno}: position {} is not a whole number", + repr(c[2]) + )) + })?; + let next = s.chroms.len() as u32; + let chrom = *s.chroms.entry(c[1].to_owned()).or_insert(next); + s.at.entry((chrom, pos)) + .or_default() + .push(s.sites.len() as u32); + s.sites.push((c[0].into(), c[3].into(), c[4].into())); + } + Ok(s) +} + +const PERSON_GENO: Spec = Spec { + verb: "person-geno", + usage: "usage: og ancient person-geno --sites SITES --genotypes GENOTYPES --snp SNP --moderns-geno MODERNS_GENO --out OUT --stats STATS\n", + options: &[ + "--sites", + "--genotypes", + "--snp", + "--moderns-geno", + "--out", + "--stats", + ], + required: &[ + "--sites", + "--genotypes", + "--snp", + "--moderns-geno", + "--out", + "--stats", + ], +}; + +fn cmd_person_geno(a: &Args, out: &mut dyn Write) -> Result<()> { + let sites = read_sites(a.req("--sites"))?; + // The person's dosage by site id: a later line for the same site wins. + let mut dos: HashMap<&str, u8> = HashMap::new(); + let (mut called, mut other) = (0u64, 0u64); + let gpath = a.req("--genotypes"); + let mut t = Text::open(gpath, Utf8::Strict)?; + let mut lineno = 0u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + continue; + } + let pos = int(c[1]).ok_or_else(|| { + Fail(format!( + "{gpath}: line {lineno}: position {} is not a whole number", + repr(c[1]) + )) + })?; + let Some(&chrom) = sites.chroms.get(c[0]) else { + continue; + }; + let Some(here) = sites.at.get(&(chrom, pos)) else { + continue; + }; + let gt = upper(c[4]); + // quirk: a position the AADR holds two SNPs at is counted once per + // SNP in sites_genotyped, as the Python counted it. + for &i in here { + let (sid, a1, a2) = &sites.sites[i as usize]; + match dosage(>, a1, a2) { + None => other += 1, + Some(d) => { + dos.insert(sid, d); + called += 1; + } + } + } + } + let snp_path = a.req("--snp"); + let geno_path = a.req("--moderns-geno"); + let mut snp = Text::open(snp_path, Utf8::Strict)?; + let mut geno = Text::open(geno_path, Utf8::Strict)?; + let mut files = Pending::new(); + let out_path = PathBuf::from(a.req("--out")); + let shown = out_path.clone(); + let o = files.create(out_path)?; + let (mut n, mut used) = (0u64, 0u64); + let mut width: Option = None; + let mut row = String::new(); + while let Some(sline) = snp.next()? { + let Some(id) = split_ws(sline).next() else { + continue; + }; + let d = dos.get(id).copied(); + let g = geno.next()?.unwrap_or(""); + if g.is_empty() { + return Err(Fail(format!("{geno_path} has fewer lines than {snp_path}"))); + } + let w = g.chars().count(); + match width { + None => width = Some(w), + Some(want) if w != want => { + return Err(Fail(format!( + "{geno_path} line {} has {w} genotypes, not {want}", + n + 1 + ))); + } + Some(_) => {} + } + if d.is_some() { + used += 1; + } + row.clear(); + row.push_str(g); + match d { + None => row.push('9'), + Some(d) => row.push(char::from(b'0' + d)), + } + row.push('\n'); + o.write_all(row.as_bytes()) + .map_err(|e| super::cannot_write(&shown, &e))?; + n += 1; + } + if geno.raw()?.is_some() { + return Err(Fail(format!("{geno_path} has more lines than {snp_path}"))); + } + let genotyped = called + other; + files.put( + PathBuf::from(a.req("--stats")), + format!( + "panel_sites\t{n}\nsites_genotyped\t{genotyped}\nother_allele\t{other}\nsites_used\t{used}\n" + ) + .as_bytes(), + )?; + files.commit_all()?; + let _ = writeln!( + out, + "Sites of the panel: {}; genotyped from the gVCF: {}; used: {} (an allele that is neither of a site's two: {})", + int_grouped(i128::from(n)), + int_grouped(i128::from(genotyped)), + int_grouped(i128::from(used)), + int_grouped(i128::from(other)) + ); + if used == 0 { + return Err(Fail( + "none of the panel's sites was genotyped: is the gVCF on GRCh38 with chr names?".into(), + )); + } + Ok(()) +} + +// --- evec, align ------------------------------------------------------------------ + +/// A smartpca `.evec`: the eigenvalues, and each row's label and PCs in +/// file order (a repeated id keeps its first place and its last values). +struct Evec { + evals: Vec, + rows: Vec<(String, String, Vec)>, + index: HashMap, +} + +impl Evec { + fn get(&self, id: &str) -> Option<(&str, &[f64])> { + self.index.get(id).map(|&i| { + let r = &self.rows[i]; + (r.1.as_str(), r.2.as_slice()) + }) + } +} + +fn read_evec(path: &str) -> Result { + let mut e = Evec { + evals: Vec::new(), + rows: Vec::new(), + index: HashMap::new(), + }; + let p = Path::new(path); + let mut t = Text::open(path, Utf8::Strict)?; + while let Some(line) = t.next()? { + let c: Vec<&str> = split_ws(line).collect(); + let Some(first) = c.first() else { continue }; + if first.starts_with('#') { + if *first == "#eigvals:" { + e.evals = c[1..] + .iter() + .map(|x| need_float(x, p)) + .collect::>()?; + } + continue; + } + let label = c[c.len() - 1].to_owned(); + let pcs: Vec = if c.len() > 1 { + c[1..c.len() - 1] + .iter() + .map(|x| need_float(x, p)) + .collect::>()? + } else { + Vec::new() + }; + match e.index.get(*first) { + Some(&i) => e.rows[i] = ((*first).to_owned(), label, pcs), + None => { + e.index.insert((*first).to_owned(), e.rows.len()); + e.rows.push(((*first).to_owned(), label, pcs)); + } + } + } + if e.evals.is_empty() || e.rows.is_empty() { + return Err(Fail(format!( + "{path} is not a smartpca .evec file (no #eigvals: line or no rows)" + ))); + } + Ok(e) +} + +/// The Pearson correlation of two equal-length series, 0 when either is +/// flat; means by `fmean`, sums by the builtin `sum`. +fn correlation(x: &[f64], y: &[f64]) -> f64 { + let mx = pycompat::stats::fmean(x.iter().copied()).unwrap_or(f64::NAN); + let my = pycompat::stats::fmean(y.iter().copied()).unwrap_or(f64::NAN); + let sxy = pycompat::num::sum(x.iter().zip(y).map(|(a, b)| (a - mx) * (b - my))); + let sx = pycompat::num::sum(x.iter().map(|a| (a - mx) * (a - mx))).sqrt(); + let sy = pycompat::num::sum(y.iter().map(|b| (b - my) * (b - my))).sqrt(); + if sx > 0.0 && sy > 0.0 { + sxy / (sx * sy) + } else { + 0.0 + } +} + +/// The k-th PC of a row, or the error that names the row. +fn pc(pcs: &[f64], k: usize, id: &str, path: &str) -> Result { + pcs.get(k) + .copied() + .ok_or_else(|| Fail(format!("{path}: {id} has no PC{}", k + 1))) +} + +/// The sign that turns each of the run's PCs into the panel's, from the +/// present-day people both runs used to build the axes. +fn axis_signs( + panel: &Evec, + panel_path: &str, + run: &Evec, + run_path: &str, + n_pcs: usize, +) -> Result> { + let mut shared: Vec<&str> = run + .rows + .iter() + .filter(|(id, lab, _)| { + lab != ANCIENT_LABEL + && lab != PERSON_LABEL + && panel.get(id).is_some_and(|(pl, _)| pl == lab) + }) + .map(|(id, ..)| id.as_str()) + .collect(); + shared.sort_unstable(); + if shared.len() < 10 { + return Err(Fail(format!( + "only {} present-day people are in both runs; the panel and this run do not match", + shared.len() + ))); + } + let mut signs = Vec::with_capacity(n_pcs); + for k in 0..n_pcs { + let mut x = Vec::with_capacity(shared.len()); + let mut y = Vec::with_capacity(shared.len()); + for id in &shared { + x.push(pc(panel.get(id).expect("shared").1, k, id, panel_path)?); + y.push(pc(run.get(id).expect("shared").1, k, id, run_path)?); + } + let r = correlation(&x, &y); + if r.abs() < MIN_AXIS_CORRELATION { + return Err(Fail(format!( + "PC{} of this run correlates {} with the panel's over {} present-day people: \ + the axes differ. Install the panel again (setup.sh --ancient-panel).", + k + 1, + fixed(r, 4), + shared.len() + ))); + } + signs.push(if r > 0.0 { 1.0 } else { -1.0 }); + } + Ok(signs) +} + +const ALIGN: Spec = Spec { + verb: "align", + usage: "usage: og ancient align --panel-evec PANEL_EVEC --run-evec RUN_EVEC [--pcs PCS]\n", + options: &["--panel-evec", "--run-evec", "--pcs"], + required: &["--panel-evec", "--run-evec"], +}; + +fn cmd_align(a: &Args, out: &mut dyn Write) -> Result<()> { + let (pp, rp) = (a.req("--panel-evec"), a.req("--run-evec")); + let panel = read_evec(pp)?; + let run = read_evec(rp)?; + let pcs = a.int("--pcs", DISTANCE_PCS).unwrap_or(DISTANCE_PCS); + let signs = axis_signs(&panel, pp, &run, rp, usize::try_from(pcs).unwrap_or(0))?; + let words: Vec = signs + .iter() + .enumerate() + .map(|(i, s)| format!("PC{}{}", i + 1, if *s > 0.0 { '+' } else { '-' })) + .collect(); + let _ = writeln!(out, "Axes agree with the panel's: {}", words.join(" ")); + Ok(()) +} + +// --- report ----------------------------------------------------------------------- + +/// A whole float as Python prints the int of it. +fn whole(x: f64) -> String { + int_str(x) +} + +/// `int(round(x))` printed. +fn rounded(x: f64) -> String { + whole(round_int(x).unwrap_or(f64::NAN)) +} + +/// A date in years before 1950 as `N BCE` or `N CE`, the AADR's own +/// reading: BCE = BP - 1950 and CE = 1950 - BP, with 1950 BP as 1 BCE. +fn year_label(bp: f64) -> String { + let y = 1950.0 - round_int(bp).unwrap_or(0.0); + if y >= 1.0 { + format!("{} CE", whole(y)) + } else { + format!("{} BCE", whole(if -y > 1.0 { -y } else { 1.0 })) + } +} + +/// The span of a population's dates, oldest first: `5300-4900 BCE`, +/// `200-450 CE`, or `300 BCE-200 CE`. +fn range_label(old_bp: f64, young_bp: f64) -> String { + let (a, b) = (year_label(old_bp), year_label(young_bp)); + if a == b { + return a; + } + let (na, ea) = a.split_once(' ').expect("N ERA"); + let (nb, eb) = b.split_once(' ').expect("N ERA"); + if ea == eb { + format!("{na}-{nb} {ea}") + } else { + format!("{a}-{b}") + } +} + +/// An `ancients.tsv` row as `csv.DictReader` gives it: a short row's +/// missing cells are `None`. +struct AncientRow { + group: String, + date_bp: Option, + country: Option, + locality: Option, +} + +/// `ancients.tsv` by id, in file order (a repeated id keeps its first +/// place and its last values). +fn read_ancients(path: &str) -> Result> { + let mut t = Text::open(path, Utf8::Strict)?; + let mut rows: Vec<(String, AncientRow)> = Vec::new(); + let mut index: HashMap = HashMap::new(); + // DictReader: the first non-empty row is the header. + let mut names: Option> = None; + while let Some(line) = t.next()? { + if line.is_empty() { + continue; + } + let cells: Vec<&str> = line.split('\t').collect(); + let Some(names) = names.as_ref() else { + names = Some(cells.iter().map(|s| (*s).to_owned()).collect()); + continue; + }; + // The last column of a repeated name wins, as in a dict. + let field = |key: &str| -> Option { + let k = names.iter().rposition(|n| n == key)?; + cells.get(k).map(|s| (*s).to_owned()) + }; + if !names.iter().any(|n| n == "id") { + return Err(Fail(format!("{path} has no id column"))); + } + let id = field("id").unwrap_or_else(|| "None".to_owned()); + let r = AncientRow { + group: field("group").unwrap_or_else(|| "None".to_owned()), + date_bp: field("date_bp"), + country: field("country"), + locality: field("locality"), + }; + match index.get(&id) { + Some(&i) => rows[i].1 = r, + None => { + index.insert(id.clone(), rows.len()); + rows.push((id, r)); + } + } + } + Ok(rows) +} + +/// The most common non-empty value other than `.`, the first in sort +/// order on a tie; `.` when there is none. +fn most_common<'a>(values: impl Iterator>) -> String { + let mut counts: BTreeMap<&str, u64> = BTreeMap::new(); + for v in values.flatten() { + if !v.is_empty() && v != "." { + *counts.entry(v).or_default() += 1; + } + } + let top = counts.values().copied().max().unwrap_or(0); + counts + .into_iter() + .find(|(_, k)| *k == top) + .map_or_else(|| ".".to_owned(), |(v, _)| v.to_owned()) +} + +/// One ancient group, nearest first. +struct Near { + rank: usize, + group: String, + n: usize, + distance: f64, + pc1: f64, + pc2: f64, + date_bp: Option, + dates: String, + country: String, + locality: String, +} + +/// The ancient groups ranked by the weighted distance from the person to +/// the group's mean position on the first `n_pcs` PCs; the top [`TOP`] and +/// the number of groups. +fn nearest_populations( + person: &[f64], + panel: &Evec, + ancients: &[(String, AncientRow)], + n_pcs: usize, +) -> Result<(Vec, usize)> { + let w: Vec = panel + .evals + .iter() + .take(n_pcs) + .map(|v| if 0.0 > *v { 0.0 } else { *v }.sqrt()) + .collect(); + // Groups in first-seen order, each with its members. + let mut order: Vec<&str> = Vec::new(); + let mut groups: HashMap<&str, Vec<(&[f64], &AncientRow)>> = HashMap::new(); + for (gid, r) in ancients { + if let Some((label, pcs)) = panel.get(gid) + && label == ANCIENT_LABEL + { + let g = groups.entry(r.group.as_str()).or_insert_with(|| { + order.push(r.group.as_str()); + Vec::new() + }); + g.push((pcs, r)); + } + } + let mut rows = Vec::with_capacity(order.len()); + for g in order { + let members = &groups[g]; + let k = members[0].0.len().min(n_pcs); + let mut centre = Vec::with_capacity(k); + for i in 0..k { + let mut xs = Vec::with_capacity(members.len()); + for (pcs, _) in members { + xs.push(*pcs.get(i).ok_or_else(|| { + Fail(format!( + "a member of the ancient group {g} has no PC{}", + i + 1 + )) + })?); + } + centre.push(pycompat::stats::fmean(xs).map_err(|e| Fail(e.to_string()))?); + } + let Some(&pc1) = centre.first() else { + return Err(Fail(format!("the ancient group {g} has no PCs"))); + }; + let distance = pycompat::num::sum(person.iter().take(n_pcs).zip(¢re).zip(&w).map( + |((a, b), wi)| { + let d = wi * (a - b); + d * d + }, + )) + .sqrt(); + let dates: Vec = members + .iter() + .filter_map(|(_, r)| r.date_bp.as_deref().and_then(to_float)) + .collect(); + let (date_bp, dates_text) = if dates.is_empty() { + (None, ".".to_owned()) + } else { + let old = dates + .iter() + .copied() + .fold(dates[0], |m, d| if d > m { d } else { m }); + let young = dates + .iter() + .copied() + .fold(dates[0], |m, d| if d < m { d } else { m }); + ( + Some( + pycompat::stats::median(dates.iter().copied()) + .map_err(|e| Fail(e.to_string()))?, + ), + range_label(old, young), + ) + }; + rows.push(Near { + rank: 0, + group: g.to_owned(), + n: members.len(), + distance, + pc1, + pc2: centre.get(1).copied().unwrap_or(0.0), + date_bp, + dates: dates_text, + country: most_common(members.iter().map(|(_, r)| r.country.as_deref())), + locality: most_common(members.iter().map(|(_, r)| r.locality.as_deref())), + }); + } + rows.sort_by(|a, b| { + a.distance + .partial_cmp(&b.distance) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.group.cmp(&b.group)) + }); + for (i, r) in rows.iter_mut().enumerate() { + r.rank = i + 1; + } + let n = rows.len(); + rows.truncate(TOP); + Ok((rows, n)) +} + +/// Age bins of the plot: (oldest BP, colour, legend), oldest first. +const AGE_BINS: [(f64, &str, &str); 4] = [ + (f64::INFINITY, "#5aa9e6", "before 8000 BCE"), + (9950.0, "#7fc97f", "8000-4000 BCE"), + (5950.0, "#f0a35e", "4000-1000 BCE"), + (2950.0, "#e377c2", "after 1000 BCE"), +]; + +fn age_colour(bp: Option) -> &'static str { + let mut colour = AGE_BINS[0].1; + for (limit, c, _) in AGE_BINS { + if bp.is_some_and(|b| b < limit) { + colour = c; + } + } + colour +} + +/// Python's `min()` and `max()` of a non-empty list of floats. +fn py_min_max(xs: &[f64]) -> (f64, f64) { + let (mut lo, mut hi) = (xs[0], xs[0]); + for &x in &xs[1..] { + if x < lo { + lo = x; + } + if x > hi { + hi = x; + } + } + (lo, hi) +} + +/// The PC1/PC2 plane: present-day people grey, ancient people coloured by +/// age, the person a large ring, on a dark background. +fn svg_plot( + person: &[f64], + panel: &Evec, + ancients: &HashMap, + sample: &str, + nearest: &[Near], +) -> Result { + const WIDTH: i64 = 900; + const HEIGHT: i64 = 720; + const PAD: f64 = 0.04; + const LEFT: i64 = 60; + const RIGHT: i64 = 20; + const TOPM: i64 = 50; + const BOTTOM: i64 = 110; + let two = |id: &str, pcs: &[f64]| -> Result<(f64, f64)> { + match pcs { + [x, y, ..] => Ok((*x, *y)), + _ => Err(Fail(format!("{id} has fewer than two PCs to plot"))), + } + }; + let mut xs = Vec::with_capacity(panel.rows.len() + 1); + let mut ys = Vec::with_capacity(panel.rows.len() + 1); + for (id, _, pcs) in &panel.rows { + let (x, y) = two(id, pcs)?; + xs.push(x); + ys.push(y); + } + let (px0, py0) = two(sample, person)?; + xs.push(px0); + ys.push(py0); + let (x0, x1) = py_min_max(&xs); + let (y0, y1) = py_min_max(&ys); + let or_one = |d: f64| if d == 0.0 { 1.0 } else { d }; + let (dx, dy) = (or_one(x1 - x0), or_one(y1 - y0)); + let (pw, ph) = (WIDTH - LEFT - RIGHT, HEIGHT - TOPM - BOTTOM); + let (left, top, pwf, phf) = (LEFT as f64, TOPM as f64, pw as f64, ph as f64); + let x = |v: f64| left + ((v - x0) / dx * (1.0 - 2.0 * PAD) + PAD) * pwf; + let y = |v: f64| top + (1.0 - ((v - y0) / dy * (1.0 - 2.0 * PAD) + PAD)) * phf; + let e = |s: &str| html_escape(s).into_owned(); + let mut out = vec![ + format!( + "" + ), + "" + .to_owned(), + format!(""), + format!( + "{} among ancient West Eurasians (AADR, projected onto present-day PCs)", + e(sample) + ), + format!( + "" + ), + ]; + for (_, label, pcs) in &panel.rows { + if label != ANCIENT_LABEL { + out.push(format!( + "", + fixed(x(pcs[0]), 1), + fixed(y(pcs[1]), 1) + )); + } + } + for (gid, label, pcs) in &panel.rows { + if label == ANCIENT_LABEL + && let Some(r) = ancients.get(gid) + { + let c = age_colour(r.date_bp.as_deref().and_then(to_float)); + out.push(format!( + "", + fixed(x(pcs[0]), 1), + fixed(y(pcs[1]), 1) + )); + } + } + for r in nearest.iter().take(5) { + out.push(format!( + "{}", + fixed(x(r.pc1) + 6.0, 1), + fixed(y(r.pc2) - 4.0, 1), + e(&r.group) + )); + } + let (px, py) = (x(px0), y(py0)); + out.push(format!( + "", + fixed(px, 1), + fixed(py, 1) + )); + out.push(format!( + "", + fixed(px, 1), + fixed(py, 1) + )); + out.push(format!( + "{}", + fixed(px + 12.0, 1), + fixed(py + 4.0, 1), + e(sample) + )); + out.push(format!( + "PC1", + fixed(left + pwf / 2.0, 0), + TOPM + ph + 22 + )); + let mid = fixed(top + phf / 2.0, 0); + out.push(format!( + "PC2" + )); + let (mut lx, ly) = (LEFT, TOPM + ph + 48); + out.push(format!( + "present day", + lx + 5, + ly - 4, + lx + 14 + )); + lx += 120; + for (_, c, name) in AGE_BINS { + out.push(format!( + "{}", + lx + 5, + ly - 4, + lx + 14, + e(name) + )); + lx += 150; + } + out.push(format!( + "{}", + ly + 26, + e(CAVEAT) + )); + out.push("".to_owned()); + Ok(out.join("\n") + "\n") +} + +const REPORT: Spec = Spec { + verb: "report", + usage: "usage: og ancient report --panel-evec PANEL_EVEC --run-evec RUN_EVEC --ancients ANCIENTS --sample SAMPLE --release RELEASE [--stats STATS] --out-tsv OUT_TSV --out-info OUT_INFO --out-txt OUT_TXT --out-svg OUT_SVG\n", + options: &[ + "--panel-evec", + "--run-evec", + "--ancients", + "--sample", + "--release", + "--stats", + "--out-tsv", + "--out-info", + "--out-txt", + "--out-svg", + ], + required: &[ + "--panel-evec", + "--run-evec", + "--ancients", + "--sample", + "--release", + "--out-tsv", + "--out-info", + "--out-txt", + "--out-svg", + ], +}; + +fn cmd_report(a: &Args, out: &mut dyn Write) -> Result<()> { + let (pp, rp) = (a.req("--panel-evec"), a.req("--run-evec")); + let panel = read_evec(pp)?; + let run = read_evec(rp)?; + let sample = a.req("--sample"); + let Some((PERSON_LABEL, pcs)) = run.get(sample) else { + return Err(Fail(format!( + "{rp} has no row for {sample} labelled {PERSON_LABEL}" + ))); + }; + let n_pcs = (DISTANCE_PCS as usize) + .min(panel.evals.len()) + .min(pcs.len()); + let signs = axis_signs(&panel, pp, &run, rp, n_pcs.max(2))?; + let person: Vec = signs.iter().zip(pcs).map(|(s, v)| s * v).collect(); + let ancients = read_ancients(a.req("--ancients"))?; + let (nearest, n_groups) = nearest_populations(&person, &panel, &ancients, n_pcs)?; + let mut stats: HashMap = HashMap::new(); + if let Some(sp) = a.get("--stats").filter(|s| !s.is_empty()) { + let mut t = Text::open(sp, Utf8::Strict)?; + while let Some(line) = t.next()? { + let (k, v) = line.split_once('\t').unwrap_or((line, "")); + stats.insert(k.to_owned(), v.to_owned()); + } + } + let stat = |k: &str| stats.get(k).map_or(".", String::as_str).to_owned(); + let n_anc = panel.rows.iter().filter(|r| r.1 == ANCIENT_LABEL).count(); + let n_mod = panel + .rows + .iter() + .filter(|r| r.1 != ANCIENT_LABEL && r.1 != PERSON_LABEL) + .count(); + let mut files = Pending::new(); + let mut tsv = + String::from("rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"); + for r in &nearest { + tsv += &format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + r.rank, + clean(&r.group), + r.n, + fixed(r.distance, 5), + fixed(r.pc1, 5), + fixed(r.pc2, 5), + r.date_bp.map_or_else(|| ".".to_owned(), rounded), + r.dates, + clean(&r.country), + clean(&r.locality) + ); + } + files.put(PathBuf::from(a.req("--out-tsv")), tsv.as_bytes())?; + let release = a.req("--release"); + let p1 = person.first().copied().unwrap_or(f64::NAN); + let p2 = person.get(1).copied().unwrap_or(f64::NAN); + let info: [(&str, String); 9] = [ + ("release", release.to_owned()), + ("sites_used", stat("sites_used")), + ("panel_sites", stat("panel_sites")), + ("present_day_people", n_mod.to_string()), + ("ancient_people", n_anc.to_string()), + ("ancient_groups", n_groups.to_string()), + ("pcs_in_distance", n_pcs.to_string()), + ("pc1", fixed(p1, 5)), + ("pc2", fixed(p2, 5)), + ]; + let mut text = String::from("key\tvalue\n"); + for (k, v) in &info { + text += &format!("{k}\t{v}\n"); + } + files.put(PathBuf::from(a.req("--out-info")), text.as_bytes())?; + let by_id: HashMap = + ancients.iter().map(|(id, r)| (id.clone(), r)).collect(); + let svg = svg_plot(&person, &panel, &by_id, sample, &nearest)?; + let mut lines = vec![ + format!( + "Ancient populations nearest to {sample} (AADR {release}, smartpca with lsqproject)" + ), + format!( + "Sites used: {} of {}; axes from {n_mod} present-day West Eurasians; {n_anc} ancient people in {n_groups} groups projected.", + stat("sites_used"), + stat("panel_sites") + ), + format!( + "Distance: the first {n_pcs} PCs, each weighted by the square root of its eigenvalue." + ), + String::new(), + ]; + for r in nearest.iter().take(10) { + lines.push(format!( + " {}. {} {} {} n={} d={}", + rjust(&r.rank.to_string(), 2), + ljust(&r.group, 45), + ljust(&r.dates, 22), + ljust(&r.country, 20), + ljust(&r.n.to_string(), 4), + fixed(r.distance, 4) + )); + } + lines.extend([String::new(), CAVEAT.to_owned(), QPADM_NOTE.to_owned()]); + let body = lines.join("\n") + "\n"; + files.put(PathBuf::from(a.req("--out-svg")), svg.as_bytes())?; + files.put(PathBuf::from(a.req("--out-txt")), body.as_bytes())?; + files.commit_all()?; + let _ = out.write_all(body.as_bytes()); + Ok(()) +} + +// --- the command line ------------------------------------------------------------- + +const USAGE: &str = "usage: og ancient {select,lift,person-geno,align,report} ...\n"; + +type Run = fn(&Args, &mut dyn Write) -> Result<()>; + +/// `og ancient [args]`: the exit code. Results go to `out`, +/// messages to `err`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og ancient: error: the following arguments are required: verb" + ); + return 2; + }; + let (spec, run): (&Spec, Run) = match verb.as_str() { + "select" => (&SELECT, cmd_select), + "lift" => (&LIFT, cmd_lift), + "person-geno" => (&PERSON_GENO, cmd_person_geno), + "align" => (&ALIGN, cmd_align), + "report" => (&REPORT, cmd_report), + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og ancient: error: argument verb: invalid choice: {}", + repr(other) + ); + return 2; + } + }; + let parsed = Args::parse(spec, &args[1..]).and_then(|a| { + for name in ["--min-snps", "--pcs"] { + if spec.options.contains(&name) { + a.int(name, 0)?; + } + } + Ok(a) + }); + let a = match parsed { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", spec.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", spec.usage); + let _ = writeln!(err, "og ancient {}: error: {msg}", spec.verb); + return 2; + } + }; + match run(&a, out) { + Ok(()) => 0, + Err(Fail(msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + 1 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dosages() { + assert_eq!(dosage("A/G", "A", "G"), Some(1)); + assert_eq!(dosage("A|A", "A", "G"), Some(2)); + assert_eq!(dosage("G/G", "A", "G"), Some(0)); + assert_eq!(dosage("A/T", "A", "G"), None); + assert_eq!(dosage("./.", "A", "G"), None); + assert_eq!(dosage("A", "A", "G"), None); + } + + #[test] + fn year_labels() { + assert_eq!(year_label(4250.0), "2300 BCE"); + assert_eq!(year_label(1950.0), "1 BCE"); + assert_eq!(year_label(1949.0), "1 CE"); + assert_eq!(year_label(1750.5), "200 CE"); + assert_eq!(range_label(7250.0, 6850.0), "5300-4900 BCE"); + assert_eq!(range_label(1750.0, 1500.0), "200-450 CE"); + assert_eq!(range_label(2250.0, 1750.0), "300 BCE-200 CE"); + assert_eq!(range_label(2250.0, 2250.0), "300 BCE"); + } + + #[test] + fn most_common_breaks_ties_by_name() { + let v = [ + Some("b"), + Some("a"), + Some("b"), + Some("a"), + Some("."), + None, + Some(""), + ]; + assert_eq!(most_common(v.into_iter()), "a"); + assert_eq!(most_common([Some("."), None].into_iter()), "."); + } + + #[test] + fn exact_int_compare() { + assert!(float_ge_int(100_000.0, 100_000)); + assert!(!float_ge_int(99_999.5, 100_000)); + assert!(!float_ge_int( + 9_007_199_254_740_992.0, + 9_007_199_254_740_993 + )); + assert!(!float_ge_int(f64::NAN, 0)); + } + + #[test] + fn clean_cells() { + assert_eq!(clean(" a\t b "), "a b"); + assert_eq!(clean(" \t "), "."); + } +} diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs new file mode 100644 index 000000000..37bdf7624 --- /dev/null +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -0,0 +1,1728 @@ +//! Step 43, archaic ancestry (`docs/43-archaic.md`): the commands around +//! IBDmix (Chen et al. 2020), as `og archaic `. +//! +//! IBDmix compares a modern genome with an archaic one site by site and +//! adds up a log-odds score (LOD) that a stretch of the modern genome shares +//! a recent ancestor with the archaic genome. It takes each site's allele +//! frequency from the modern people in its input, and one genome alone +//! gives no frequency, so `genotypes` adds stand-in columns whose genotypes +//! carry the 1000 Genomes allele count of the chosen population. +//! +//! - `table` (setup): one GRCh37 chromosome of the archaic table, a +//! sites-only VCF that Picard lifts to GRCh38. The 1000 Genomes SNVs and +//! the three archaic genomes' calls are merged by position in one pass; +//! each genome is read inside its own mask, and a call outside it (or at +//! an excluded site) is kept as masked, code 10 plus its ALT count. +//! - `strict` (setup): the lifted rows of one chromosome, once per +//! position, with every call outside the 1000 Genomes strict mask marked +//! masked. +//! - `genotypes` (step 43): IBDmix's genotype file of each chromosome, each +//! archaic genome's mask of masked positions, and the spans table. +//! - `report` (step 43): the segments at LOD 4 and 50 kb or longer, merged +//! into Neanderthal and Denisovan ones, the shares, the ten longest and +//! their protein-coding genes. +//! +//! Each command writes what `bin/archaic_report.py` wrote (checked by +//! `tests/parity`). Deliberate changes: every output is written under a +//! temporary name and renamed into place; a gzip input is told by its first +//! bytes, not its name; `strict` reads its rows in position order and stops +//! (exit 2) on a row that goes back, instead of keeping every position of a +//! chromosome in memory; and malformed input is an `ERROR:` line, never a +//! traceback. +//! +//! The table's bytes are part of the archaic data version: a change to what +//! `table` or `strict` writes must change `ARCHAIC_TABLE_FORMAT` in +//! `scripts/setup.sh`, so existing installs rebuild their table. + +use super::{ArgError, Args, Fail, Result, Spec, cannot_read, cannot_write, int, repr, split_ws}; +use flate2::Compression; +use flate2::write::GzEncoder; +use og_io::{AtomicFile, Lines, Utf8}; +use pycompat::fmt::{fixed, general, zero_pad}; +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::io::{self, BufRead, Write}; +use std::path::{Path, PathBuf}; + +pub const ARCHAICS: [&str; 3] = ["Altai", "Vindija", "Denisova"]; +const NEANDERTHALS: [&str; 2] = ["Altai", "Vindija"]; +const POPS: [&str; 6] = ["ALL", "AFR", "AMR", "EAS", "EUR", "SAS"]; +const TABLE_HEADER: [&str; 13] = [ + "chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "AF_ALL", "AF_AFR", "AF_AMR", + "AF_EAS", "AF_EUR", "AF_SAS", +]; +/// Table code: MASKED plus the ALT count, a call outside its genome's mask. +const MASKED: u8 = 10; +/// Stand-in modern people: 200 alleles, about one 1000 Genomes population. +const PSEUDO: i64 = 100; +/// Chen et al. 2020: LOD 4 and 50 kb. +const MIN_LOD: f64 = 4.0; +const MIN_LENGTH: i64 = 50_000; +const TOP: usize = 10; +const SEGMENT_HEADER: &str = "chrom\tstart\tend\tlength_bp\tancestry\tmatched\tlod\tgenes"; +const CAVEAT: &str = "IBDmix finds stretches where your genome matches a Neanderthal or the Denisovan genome more \ +closely than the allele frequencies of a modern population explain. A stretch counts when either \ +of your two copies carries it, and the share is of the analysed span of the autosomes. Some of what \ +it finds in anyone is old ancestry shared with the archaic genomes rather than gene flow from them: \ +Chen et al. 2020 find Neanderthal sequence in every population they studied, Africans included. \ +The numbers move with the cutoffs (LOD 4, 50 kb) and with the frequencies used. An estimate, not \ +a measurement."; + +/// Why a command stopped: its exit code (2 for a wrong option value the +/// command checks itself, 1 otherwise) and the message after `ERROR: `. +#[derive(Debug)] +struct Stop(u8, String); + +impl From for Stop { + fn from(f: Fail) -> Self { + Stop(1, f.0) + } +} + +fn die(msg: impl Into) -> std::result::Result { + Err(Stop(1, msg.into())) +} + +fn usage(msg: impl Into) -> std::result::Result { + Err(Stop(2, msg.into())) +} + +type Out = std::result::Result; + +/// A text file read line by line as Python's text mode reads it (universal +/// newlines, strict UTF-8; gzip read through), keeping the raw line for +/// messages that quote it. +struct Text { + path: PathBuf, + lines: Lines>, + line: String, +} + +impl Text { + fn open(path: &str) -> Result { + let path = PathBuf::from(path); + let input = og_io::open(&path).map_err(|e| cannot_read(&path, &e))?; + Ok(Text { + path, + lines: Lines::new(input, Utf8::Strict), + line: String::new(), + }) + } + + /// The next line with its `\n`, or `None` at the end. + fn raw(&mut self) -> Result> { + let more = self + .lines + .read_line(&mut self.line) + .map_err(|e| cannot_read(&self.path, &e))?; + Ok(more.then_some(self.line.as_str())) + } + + /// The next line without its `\n` (`line.rstrip("\n")`). + fn next(&mut self) -> Result> { + Ok(self.raw()?.map(|l| l.strip_suffix('\n').unwrap_or(l))) + } +} + +/// Python's `int(s)`, or the message of the ValueError it raises. +fn need_int(s: &str) -> Out { + int(s).ok_or_else(|| { + Stop( + 1, + format!("invalid literal for int() with base 10: {}", repr(s)), + ) + }) +} + +/// Python's `float(s)`, or the message of the ValueError it raises. +fn need_float(s: &str) -> Out { + pycompat::num::parse_float(s) + .map_err(|_| Stop(1, format!("could not convert string to float: {}", repr(s)))) +} + +/// `repr(line[:80])`. +fn head80(line: &str) -> String { + let end = line.char_indices().nth(80).map_or(line.len(), |(i, _)| i); + repr(&line[..end]) +} + +/// `os.path.join(a, b)` for a relative `b`. +fn join(a: &str, b: &str) -> String { + if a.is_empty() || a.ends_with('/') { + format!("{a}{b}") + } else { + format!("{a}/{b}") + } +} + +/// `'0/1'`, `'1|1'`, `'0/0'` to 1, 2, 0; a missing, partial or +/// multi-allelic call to `None`. +fn gt_code(gt: &str) -> Option { + let mut parts = gt.split(['/', '|']); + let (Some(a), Some(b), None) = (parts.next(), parts.next(), parts.next()) else { + return None; + }; + let allele = |x: &str| match x { + "0" => Some(0), + "1" => Some(1), + _ => None, + }; + Some(allele(a)? + allele(b)?) +} + +/// Python's `max(a, b)` and `min(a, b)` of floats: the first unless the +/// second compares greater (smaller), so a NaN first stays. +fn py_max(a: f64, b: f64) -> f64 { + if b > a { b } else { a } +} + +fn py_min(a: f64, b: f64) -> f64 { + if b < a { b } else { a } +} + +// --- table (setup) ---------------------------------------------------------------- + +/// One mask's intervals on one chromosome, sorted and merged: 0-based, +/// half-open, so `[s, e)` holds the 1-based positions `s + 1` to `e`. The +/// chromosome is matched with and without the `chr` prefix. +fn read_mask(path: &str, chrom: &str) -> Out> { + let num = chrom.strip_prefix("chr").unwrap_or(chrom); + let with_chr = format!("chr{num}"); + let mut iv = Vec::new(); + let mut t = Text::open(path)?; + while let Some(line) = t.raw()? { + if line.starts_with('#') || line.starts_with("track") || line.starts_with("browser") { + continue; + } + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 3 || (c[0] != num && c[0] != with_chr) { + continue; + } + let (s, e) = (need_int(c[1])?, need_int(c[2])?); + if e <= s { + continue; // an empty or reversed interval holds no position + } + iv.push((s, e)); + } + iv.sort_unstable(); + let mut out: Vec<(i64, i64)> = Vec::with_capacity(iv.len()); + for (s, e) in iv { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + Ok(out) +} + +/// Whether a 1-based position is inside a mask, for positions asked in +/// ascending order: the walk over the intervals never goes back. +struct Mask { + iv: Vec<(i64, i64)>, + i: usize, +} + +impl Mask { + fn holds(&mut self, pos: i64) -> bool { + let p = pos - 1; + while self.i < self.iv.len() && self.iv[self.i].1 <= p { + self.i += 1; + } + self.i < self.iv.len() && self.iv[self.i].0 <= p + } +} + +/// One record of a table input: a 1000 Genomes SNV (REF, ALT and the six +/// frequencies as written) or an archaic call (REF, ALT, GT). +#[derive(Debug, Clone)] +enum Rec { + Modern { + r#ref: String, + alt: String, + afs: Vec, + }, + Archaic { + r#ref: String, + alt: String, + gt: String, + }, +} + +/// One input of the merge: its lines, read on demand. +struct Stream { + text: Text, + /// 0 for the 1000 Genomes lines, 1 + i for archaic genome i. + src: usize, +} + +impl Stream { + /// The next record, `(pos, rec)`; 1000 Genomes lines that are not + /// SNVs are skipped. + fn next(&mut self) -> Out> { + loop { + let path = self.text.path.display().to_string(); + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + let line = raw.strip_suffix('\n').unwrap_or(raw); + let c: Vec<&str> = line.split('\t').collect(); + if self.src == 0 { + if c.len() < 9 { + return die(format!( + "{path}: a line has {} columns, want 9: {}", + c.len(), + head80(raw) + )); + } + if c[1].chars().count() != 1 || c[2].chars().count() != 1 { + continue; + } + let pos = need_int(c[0])?; + return Ok(Some(( + pos, + Rec::Modern { + r#ref: c[1].to_owned(), + alt: c[2].to_owned(), + afs: c[3..9].iter().map(|s| (*s).to_owned()).collect(), + }, + ))); + } + if c.len() < 4 { + return die(format!( + "{path}: a line has {} columns, want 4: {}", + c.len(), + head80(raw) + )); + } + let pos = need_int(c[0])?; + return Ok(Some(( + pos, + Rec::Archaic { + r#ref: c[1].to_owned(), + alt: c[2].to_owned(), + gt: c[3].to_owned(), + }, + ))); + } + } +} + +/// An archaic call at one position: (REF, ALT, GT). +type Call<'a> = Option<(&'a str, &'a str, &'a str)>; + +/// The ALT count of one archaic record at a row with this REF and ALT, or 9. +fn archaic_code(rec: Call<'_>, r: &str, alt: &str) -> u8 { + let Some((aref, aalt, gt)) = rec else { + return 9; + }; + if aref != r { + return 9; + } + let Some(code) = gt_code(gt) else { + return 9; + }; + if aalt == "." || aalt == "<*>" || code == 0 { + return if code == 0 { 0 } else { 9 }; + } + if aalt == alt { code } else { 9 } +} + +/// The row of one position: REF, ALT, the three codes and the six +/// frequencies as written, or `None` when nothing is known there. +fn table_row<'a>( + modern: Option<(&'a str, &'a str, &'a [String])>, + arch: [Call<'a>; 3], +) -> Option<(&'a str, &'a str, [u8; 3], Vec)> { + const ZEROS: [&str; 6] = ["0"; 6]; + let (r, alt, afs): (&str, &str, Vec<&str>) = match modern { + Some((r, alt, afs)) => (r, alt, afs.iter().map(String::as_str).collect()), + None => { + // A site no modern person varies at: the ALT of the first + // archaic genome with a non-reference call, every frequency 0. + let (r, alt, _) = arch.iter().flatten().find(|(r, a, gt)| { + !matches!(gt_code(gt), None | Some(0)) + && r.chars().count() == 1 + && a.chars().count() == 1 + })?; + (r, alt, ZEROS.to_vec()) + } + }; + let codes = arch.map(|a| archaic_code(a, r, alt)); + if codes.iter().all(|&c| c == 9) { + return None; + } + if modern.is_none() && !codes.iter().any(|&c| c == 1 || c == 2) { + return None; + } + let clean = afs + .iter() + .map(|v| { + let x = pycompat::num::parse_float(v).unwrap_or(0.0); + general(py_min(py_max(x, 0.0), 1.0), 6) + }) + .collect(); + Some((r, alt, codes, clean)) +} + +/// `NAME=FILE` with NAME one of the archaic genomes. +fn named<'a>(spec: &'a str, option: &str) -> Out<(&'a str, &'a str)> { + let (name, path) = spec.split_once('=').unwrap_or((spec, "")); + if !ARCHAICS.contains(&name) || path.is_empty() { + return usage(format!( + "{option} takes NAME=FILE with NAME one of {}, got {}", + ARCHAICS.join(", "), + repr(spec) + )); + } + Ok((name, path)) +} + +fn cmd_table(a: &Args, err: &mut dyn Write) -> Out<()> { + let chrom_arg = a.req("--chrom"); + let mut names = Vec::new(); + let mut paths = Vec::new(); + for spec in a.all("--archaic") { + let (name, path) = named(spec, "--archaic")?; + names.push(name); + paths.push(path); + } + if names != ARCHAICS { + return usage(format!( + "--archaic must name {} in that order, got {}", + ARCHAICS.join(", "), + names.join(", ") + )); + } + let mut masks: [Option; 3] = [None, None, None]; + for spec in a.all("--mask") { + let (name, path) = named(spec, "--mask")?; + let k = ARCHAICS.iter().position(|n| *n == name).expect("checked"); + if masks[k].is_some() { + return usage(format!("--mask names {name} twice")); + } + let iv = read_mask(path, chrom_arg)?; + if iv.is_empty() { + return die(format!( + "{path} holds no interval on chromosome {chrom_arg}: a mask that matches nothing \ + would make every site unknown" + )); + } + masks[k] = Some(Mask { iv, i: 0 }); + } + let mut excluded = Vec::new(); + for path in a.all("--exclude") { + let iv = read_mask(path, chrom_arg)?; + if iv.is_empty() { + return die(format!( + "{path} holds no interval on chromosome {chrom_arg}: an empty exclusion list is a \ + download that went wrong, not a chromosome without segmental duplications" + )); + } + excluded.push(Mask { iv, i: 0 }); + } + let chrom = if chrom_arg.starts_with("chr") { + chrom_arg.to_owned() + } else { + format!("chr{chrom_arg}") + }; + let out_path = PathBuf::from(a.req("--out")); + let mut out = AtomicFile::create(&out_path).map_err(|e| cannot_write(&out_path, &e))?; + let mut head = format!("##fileformat=VCFv4.2\n##contig=\n"); + for n in ARCHAICS { + head += &format!( + "##INFO=\n" + ); + } + for p in POPS { + head += &format!( + "##INFO=\n" + ); + } + head += "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"; + let w = |out: &mut AtomicFile, s: &str| { + out.write_all(s.as_bytes()) + .map_err(|e| Stop::from(cannot_write(&out_path, &e))) + }; + w(&mut out, &head)?; + + // The k-way merge of heapq.merge: the head of each input on a heap + // keyed (position, input), each input advanced only when its head is + // taken, so the order is the same as the Python's on any input. + let mut streams = vec![Stream { + text: Text::open(a.req("--modern"))?, + src: 0, + }]; + for (i, p) in paths.iter().enumerate() { + streams.push(Stream { + text: Text::open(p)?, + src: 1 + i, + }); + } + let mut heads: Vec> = vec![None; streams.len()]; + let mut heap = BinaryHeap::new(); + for (i, s) in streams.iter_mut().enumerate() { + if let Some((pos, rec)) = s.next()? { + heads[i] = Some(rec); + heap.push(Reverse((pos, i))); + } + } + let mut rows: u64 = 0; + let mut cur: Option = None; + let mut modern: Option = None; + let mut arch: [Option; 3] = [None, None, None]; + let mut last = -1i64; + let mut flush = |cur: i64, + modern: &Option, + arch: &[Option; 3], + out: &mut AtomicFile| + -> Out<()> { + // Every mask is walked for every position, before any early + // return, because the walk only goes forward. + let mut excl = false; + for m in excluded.iter_mut() { + excl |= m.holds(cur); + } + let mut outside = [false; 3]; + for (k, m) in masks.iter_mut().enumerate() { + outside[k] = excl || m.as_mut().is_some_and(|m| !m.holds(cur)); + } + let modern = modern.as_ref().map(|m| match m { + Rec::Modern { r#ref, alt, afs } => (r#ref.as_str(), alt.as_str(), afs.as_slice()), + Rec::Archaic { .. } => unreachable!("only the modern input gives Modern"), + }); + let calls = [0, 1, 2].map(|k| match &arch[k] { + Some(Rec::Archaic { r#ref, alt, gt }) => { + Some((r#ref.as_str(), alt.as_str(), gt.as_str())) + } + _ => None, + }); + let Some((r, alt, codes, afs)) = table_row(modern, calls) else { + return Ok(()); + }; + // A call outside its genome's mask, or at an excluded site, is kept + // but marked masked: step 43 hands IBDmix the call and a mask of + // those positions, as Chen et al. do. + let mut info = String::new(); + for k in 0..3 { + let c = codes[k]; + let c = if outside[k] && c != 9 { c + MASKED } else { c }; + info += &format!("{}={c};", ARCHAICS[k]); + } + for (k, p) in POPS.iter().enumerate() { + info += &format!("AF_{p}={}", afs[k]); + if k + 1 < POPS.len() { + info.push(';'); + } + } + out.write_all(format!("{chrom}\t{cur}\t.\t{r}\t{alt}\t.\t.\t{info}\n").as_bytes()) + .map_err(|e| Stop::from(cannot_write(&out_path, &e)))?; + rows += 1; + Ok(()) + }; + while let Some(Reverse((pos, i))) = heap.pop() { + let rec = heads[i].take().expect("a head for each heap entry"); + if pos < last { + return die(format!("input not sorted by position at {pos}")); + } + if cur != Some(pos) { + if let Some(c) = cur { + flush(c, &modern, &arch, &mut out)?; + } + cur = Some(pos); + modern = None; + arch = [None, None, None]; + } + last = pos; + match rec { + Rec::Modern { .. } => { + // A second modern record at one position (a split + // multi-allelic site) makes the row ambiguous: keep neither. + modern = Some(match modern { + None => rec, + Some(_) => Rec::Modern { + r#ref: "N".into(), + alt: "N".into(), + afs: vec!["0".into(); POPS.len()], + }, + }); + } + Rec::Archaic { .. } => arch[streams[i].src - 1] = Some(rec), + } + // As heapq.merge does, the input whose record was taken is read on + // only now. + if let Some((p, r)) = streams[i].next()? { + heads[i] = Some(r); + heap.push(Reverse((p, i))); + } + } + if let Some(c) = cur { + flush(c, &modern, &arch, &mut out)?; + } + out.commit().map_err(|e| cannot_write(&out_path, &e))?; + let _ = writeln!(err, "{chrom}: {rows} rows"); + Ok(()) +} + +// --- strict (setup) --------------------------------------------------------------- + +/// The lifted GRCh38 rows of one chromosome (no header, the 13 table +/// columns), once per position, with the calls of every row outside the +/// 1000 Genomes strict mask marked masked (MASKED plus the ALT count; 9 +/// stays 9). The strict mask is the same for the three genomes, so all +/// three are marked. +fn cmd_strict(a: &Args, err: &mut dyn Write) -> Out<()> { + let chrom = a.req("--chrom"); + let strict = a.req("--strict"); + let iv = read_mask(strict, chrom)?; + if iv.is_empty() { + return die(format!( + "{strict} holds no interval on chromosome {chrom}: a mask that matches nothing \ + would mark every site masked" + )); + } + let rows_path = a.req("--rows"); + let mut t = Text::open(rows_path)?; + let out_path = PathBuf::from(a.req("--out")); + let mut out = AtomicFile::create(&out_path).map_err(|e| cannot_write(&out_path, &e))?; + let (mut rows, mut marked) = (0u64, 0u64); + let mut last: Option = None; + let mut n = 0u64; + while let Some(raw) = t.raw()? { + n += 1; + let line = raw.strip_suffix('\n').unwrap_or(raw); + let mut c: Vec = line.split('\t').map(str::to_owned).collect(); + if c.len() != TABLE_HEADER.len() { + return die(format!( + "{rows_path}: a row has {} columns, want {}: {}", + c.len(), + TABLE_HEADER.len(), + head80(raw) + )); + } + let pos = need_int(&c[1])?; + // The rows come sorted (Picard sorts its output), so a position + // seen before is the one just before: a repeat keeps its first row. + match last { + Some(l) if pos == l => continue, + Some(l) if pos < l => { + return usage(format!( + "{rows_path}: line {n}: position {pos} comes after {l}; the lifted rows must be sorted by position" + )); + } + _ => {} + } + last = Some(pos); + let p = pos - 1; + let i = iv.partition_point(|&(s, _)| s <= p); + let inside = i > 0 && p < iv[i - 1].1; + if !inside { + let mut hit = false; + for cell in &mut c[4..7] { + if let Some(v) = ["0", "1", "2"].iter().position(|x| x == cell) { + *cell = (v as u8 + MASKED).to_string(); + hit = true; + } + } + marked += u64::from(hit); + } + out.write_all((c.join("\t") + "\n").as_bytes()) + .map_err(|e| cannot_write(&out_path, &e))?; + rows += 1; + } + out.commit().map_err(|e| cannot_write(&out_path, &e))?; + let _ = writeln!( + err, + "{chrom}: {rows} rows, {marked} outside the strict mask and marked masked" + ); + Ok(()) +} + +// --- genotypes (step 43) ---------------------------------------------------------- + +/// The stand-in columns for each ALT allele count 0..2n: as many 2s and +/// one 1 as the count needs, then 0s. +fn pseudo_columns(n: usize) -> Vec { + (0..=2 * n) + .map(|ac| { + let mut g = vec!["2"; ac / 2]; + if ac % 2 == 1 { + g.push("1"); + } + g.resize(n, "0"); + g.join("\t") + }) + .collect() +} + +/// A call of the sample's VCF: REF, ALT (`None` for an insertion, a +/// deletion or a multi-allelic record) and the code (0, 1, 2, or 9 when it +/// cannot be read). +#[derive(Debug, Clone, PartialEq, Eq)] +struct Sample { + r#ref: String, + alt: Option, + code: u8, +} + +/// What [`pick_call`] keeps. +#[derive(Debug, PartialEq, Eq)] +enum Picked { + Call(Sample), + Clash, +} + +/// The call to keep when a position comes twice, as in a VCF split with +/// bcftools norm -m-. An SNV wins over an insertion or a multi-allelic +/// record, in either order. Of one SNV twice, a usable record wins over +/// code 9, and two different usable codes clash. Of two different SNVs, +/// the one that carries its ALT wins over a 0/0 or unusable one; both 0/0 +/// is the reference; both carrying their ALT, or neither readable, clash +/// (unknown, 9). +fn pick_call(prev: Option, call: Sample) -> Picked { + let Some(prev) = prev else { + return Picked::Call(call); + }; + if prev.alt.is_none() || call.alt.is_none() { + return Picked::Call(if prev.alt.is_none() && call.alt.is_some() { + call + } else { + prev + }); + } + if call.alt == prev.alt { + if call.code == 9 || call.code == prev.code { + return Picked::Call(prev); + } + return if prev.code == 9 { + Picked::Call(call) + } else { + Picked::Clash + }; + } + let carries = |c: u8| c == 1 || c == 2; + let (has_prev, has_call) = (carries(prev.code), carries(call.code)); + if has_prev != has_call { + return Picked::Call(if has_prev { prev } else { call }); + } + if !has_prev && prev.code == 0 && call.code == 0 { + return Picked::Call(Sample { + r#ref: prev.r#ref, + alt: None, + code: 0, + }); + } + Picked::Clash +} + +/// One record of the sample's VCF that `genotypes` reads. +struct VcfRec { + chrom: String, + pos: i64, + call: Sample, +} + +/// The biallelic SNVs (and the other records, as code 9) of a one-sample +/// VCF, in file order. A record whose FILTER is not PASS or `.` gives +/// code 9 (not read), so a DeepVariant RefCall is neither your allele nor +/// the reference. +struct Vcf { + text: Text, +} + +impl Vcf { + fn next(&mut self) -> Out> { + loop { + let path = self.text.path.display().to_string(); + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + if raw.starts_with("##") { + continue; + } + let line = raw.strip_suffix('\n').unwrap_or(raw); + let c: Vec<&str> = line.split('\t').collect(); + if raw.starts_with('#') { + if c.len() != 10 { + return die(format!( + "{path} holds {} samples; step 43 reads a one-sample VCF \ + (keep one with bcftools view -s ).", + c.len().saturating_sub(9) + )); + } + continue; + } + if c.len() < 10 || c[3].chars().count() != 1 || c[4] == "." { + continue; + } + if c[4].contains(',') || c[4].chars().count() != 1 { + return Ok(Some(VcfRec { + chrom: c[0].to_owned(), + pos: need_int(c[1])?, + call: Sample { + r#ref: c[3].to_owned(), + alt: None, + code: 9, + }, + })); + } + let fmt: Vec<&str> = c[8].split(':').collect(); + let Some(k) = fmt.iter().position(|f| *f == "GT") else { + continue; + }; + let gt = c[9].split(':').nth(k).unwrap_or("."); + let mut code = gt_code(gt).unwrap_or(9); + if c[6] != "PASS" && c[6] != "." { + code = 9; + } + return Ok(Some(VcfRec { + chrom: c[0].to_owned(), + pos: need_int(c[1])?, + call: Sample { + r#ref: c[3].to_owned(), + alt: Some(c[4].to_owned()), + code, + }, + })); + } + } +} + +/// The sample's calls on one chromosome, one per position after +/// [`pick_call`], sorted by position: a compact array, not a map. +fn settle(mut recs: Vec<(i64, Sample)>) -> Vec<(i64, Sample)> { + // A stable sort keeps the records of one position in file order. + recs.sort_by_key(|(p, _)| *p); + let mut out: Vec<(i64, Sample)> = Vec::new(); + let mut clashed = false; + for (pos, call) in recs { + match out.last_mut() { + Some((p, kept)) if *p == pos => { + if clashed { + continue; + } + let r#ref = call.r#ref.clone(); + match pick_call(Some(kept.clone()), call) { + Picked::Call(c) => *kept = c, + Picked::Clash => { + *kept = Sample { + r#ref, + alt: None, + code: 9, + }; + clashed = true; + } + } + } + _ => { + clashed = false; + out.push((pos, call)); + } + } + } + out +} + +/// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. +fn table_chrom(name: &str) -> Option<&str> { + let chrom = name.strip_suffix(".tsv.gz")?; + let num = chrom.strip_prefix("chr")?; + let ok = match num.len() { + 1 => matches!(num.as_bytes()[0], b'1'..=b'9'), + 2 => { + let (a, b) = (num.as_bytes()[0], num.as_bytes()[1]); + (a == b'1' && b.is_ascii_digit()) || (a == b'2' && (b'0'..=b'2').contains(&b)) + } + _ => false, + }; + ok.then_some(chrom) +} + +/// A file written whole through a temporary name. +fn put(path: &Path, data: &[u8]) -> Result<()> { + let mut f = AtomicFile::create(path).map_err(|e| cannot_write(path, &e))?; + f.write_all(data).map_err(|e| cannot_write(path, &e))?; + f.commit().map_err(|e| cannot_write(path, &e)) +} + +fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { + let pseudo_n = a + .int("--pseudo", PSEUDO) + .map_err(|_| Stop(2, String::new()))?; + if pseudo_n < 1 { + return usage("--pseudo must be 1 or more"); + } + let pop = a.get("--pop").unwrap_or("ALL"); + let Some(af_col) = POPS.iter().position(|p| *p == pop).map(|k| 7 + k) else { + return usage(format!( + "--pop must be one of {}, got {}", + POPS.join(", "), + repr(pop) + )); + }; + let table_dir = a.req("--table-dir"); + let rd = std::fs::read_dir(table_dir).map_err(|e| cannot_read(Path::new(table_dir), &e))?; + let mut names: Vec = Vec::new(); + for e in rd { + let e = e.map_err(|e| cannot_read(Path::new(table_dir), &e))?; + if let Ok(n) = e.file_name().into_string() { + names.push(n); + } + } + names.sort(); + let tables: Vec<(String, String)> = names + .iter() + .filter_map(|n| table_chrom(n).map(|c| (c.to_owned(), join(table_dir, n)))) + .collect(); + if tables.is_empty() { + return die(format!( + "{table_dir} holds no chrN.tsv.gz: install the archaic data with scripts/setup.sh --archaic" + )); + } + let out_dir = a.req("--out-dir"); + std::fs::create_dir_all(out_dir).map_err(|e| cannot_write(Path::new(out_dir), &e))?; + let n = pseudo_n as usize; + let pseudo = pseudo_columns(n); + let two_n = (2 * n) as f64; + let mut head = String::from("chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tSAMPLE"); + let mut samples = String::from("SAMPLE\n"); + for i in 1..=n { + let id = format!("P{}", zero_pad(i as i128, 3)); + head += &format!("\t{id}"); + samples += &format!("{id}\n"); + } + head.push('\n'); + put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; + + let vcf_path = a.req("--vcf"); + let mut vcf = Vcf { + text: Text::open(vcf_path)?, + }; + let mut spans: Vec = Vec::new(); + let mut done: Vec = Vec::new(); + let mut pending = vcf.next()?; + while let Some(first) = pending.take() { + let chrom = first.chrom.clone(); + if done.contains(&chrom) { + return die(format!( + "{vcf_path} is not sorted: {chrom} comes back after another chromosome" + )); + } + done.push(chrom.clone()); + let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); + // The sample's calls on this chromosome; on a chromosome the table + // does not hold they are read past. + let mut recs = Vec::new(); + if table.is_some() { + recs.push((first.pos, first.call)); + } + loop { + match vcf.next()? { + Some(r) if r.chrom == chrom => { + if table.is_some() { + recs.push((r.pos, r.call)); + } + } + other => { + pending = other; + break; + } + } + } + let Some(table) = table else { + continue; + }; + let calls = settle(recs); + if let Some(span) = write_chrom( + &chrom, table, &calls, &head, &pseudo, af_col, two_n, out_dir, + )? { + spans.push(span); + } + } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; + if spans.is_empty() { + let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); + let mut sorted = names.clone(); + sorted.sort_unstable(); + return die(format!( + "{vcf_path} has no records on a chromosome of the archaic table ({})", + sorted.join(", ") + )); + } + let sites: u64 = spans.iter().map(|s| s.3).sum(); + let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); + Ok(()) +} + +/// A chromosome's row of `spans.tsv`: chromosome, first and last +/// position, sites, sites with your ALT, sites with another allele. +type Span = (String, i64, i64, u64, u64, u64); + +/// IBDmix's genotype file of one chromosome, and each archaic genome's +/// mask; the chromosome's span row, or `None` when no row was written. +#[allow(clippy::too_many_arguments)] +fn write_chrom( + chrom: &str, + table: &str, + calls: &[(i64, Sample)], + head: &str, + pseudo: &[String], + af_col: usize, + two_n: f64, + out_dir: &str, +) -> Out> { + let num = &chrom[3..]; + let path = Path::new(out_dir).join(format!("{chrom}.gt.gz")); + let file = AtomicFile::create(&path).map_err(|e| cannot_write(&path, &e))?; + let mut gz = GzEncoder::new(file, Compression::new(1)); + let werr = |e: io::Error| Stop::from(cannot_write(&path, &e)); + gz.write_all(head.as_bytes()).map_err(werr)?; + let mut t = Text::open(table)?; + let header = t.next()?.unwrap_or(""); + if header.split('\t').ne(TABLE_HEADER.iter().copied()) { + return die(format!( + "{table} does not start with the table header ({})", + TABLE_HEADER.join(" ") + )); + } + let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); + // Per archaic genome, the runs of written rows whose call is masked: + // [first, last] positions, closed at the next unmasked row. + let mut runs: [Vec<(i64, i64)>; 3] = [Vec::new(), Vec::new(), Vec::new()]; + let mut open_run: [Option<(i64, i64)>; 3] = [None; 3]; + let mut row = String::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 7 || c.len() <= af_col { + return die(format!( + "{table}: a row has {} columns, want {}", + c.len(), + TABLE_HEADER.len() + )); + } + let pos = need_int(c[1])?; + let (r, alt) = (c[2], c[3]); + let mut arch = [0u8; 3]; + for k in 0..3 { + let v = c[4 + k]; + arch[k] = match v { + "0" | "1" | "2" | "9" | "10" | "11" | "12" => v.parse().expect("a code"), + _ => { + return die(format!( + "{table}: {chrom}:{pos} has {} code {}; the table holds 0, 1, 2, 9 or {MASKED} to {}", + ARCHAICS[k], + repr(v), + MASKED + 2 + )); + } + }; + } + let code = match calls.binary_search_by_key(&pos, |(p, _)| *p) { + Err(_) => 0, + Ok(i) => { + let call = &calls[i].1; + if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { + // Another allele than the row's: IBDmix's generate_gt + // drops the line. + dropped += 1; + continue; + } + call.code + } + }; + if code == 1 || code == 2 { + carried += 1; + } + let ac = match pycompat::num::parse_float(c[af_col]) { + Err(_) => 0.0, + Ok(x) => match pycompat::fmt::round_int(x * two_n) { + Ok(v) => v, + Err(pycompat::fmt::RoundError::Nan) => 0.0, + Err(e) => return die(format!("{table}: {chrom}:{pos}: {e}")), + }, + }; + let ac = ac.clamp(0.0, two_n) as usize; + for k in 0..3 { + if arch[k] >= MASKED { + match &mut open_run[k] { + None => open_run[k] = Some((pos, pos)), + Some(run) => run.1 = pos, + } + } else if let Some(run) = open_run[k].take() { + runs[k].push(run); + } + } + row.clear(); + row += &format!("{num}\t{pos}\t{r}\t{alt}"); + for v in arch { + let v = if v >= MASKED { v - MASKED } else { v }; + row += &format!("\t{v}"); + } + row += &format!("\t{code}\t{}\n", pseudo[ac]); + gz.write_all(row.as_bytes()).map_err(werr)?; + if n == 0 { + first = pos; + } + last = pos; + n += 1; + } + if n == 0 { + drop(gz); + match std::fs::remove_file(&path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(Stop::from(cannot_write(&path, &e))); + } + _ => {} + } + return Ok(None); + } + gz.finish().and_then(AtomicFile::commit).map_err(werr)?; + // IBDmix's mask (-r): the chromosome as in the genotype file, and a + // line covers the positions start + 1 to end, as in a BED. + for k in 0..3 { + if let Some(run) = open_run[k] { + runs[k].push(run); + } + let text: String = runs[k] + .iter() + .map(|(s, e)| format!("{num}\t{}\t{e}\n", s - 1)) + .collect(); + put( + &Path::new(out_dir).join(format!("{chrom}.{}.mask.bed", ARCHAICS[k])), + text.as_bytes(), + )?; + } + Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) +} + +// --- report (step 43) ------------------------------------------------------------- + +/// A segment: chromosome, start, end, LOD. +type Seg = (String, i64, i64, f64); + +/// IBDmix's output rows for SAMPLE, kept at LOD 4 and 50 kb or longer. +fn read_segments(path: &str) -> Out> { + let mut t = Text::open(path)?; + let head: Vec = t + .next()? + .unwrap_or("") + .split('\t') + .map(str::to_owned) + .collect(); + let first5: Vec<&str> = head.iter().take(5).map(String::as_str).collect(); + if first5 != ["ID", "chrom", "start", "end", "slod"] { + return die(format!( + "{path} is not IBDmix output (header {})", + first5.join(" ") + )); + } + let mut segs = Vec::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 || c[0] != "SAMPLE" { + continue; + } + let (s, e, lod) = (need_int(c[2])?, need_int(c[3])?, need_float(c[4])?); + if lod >= MIN_LOD && e - s >= MIN_LENGTH { + let chrom = if c[1].starts_with("chr") { + c[1].to_owned() + } else { + format!("chr{}", c[1]) + }; + segs.push((chrom, s, e, lod)); + } + } + Ok(segs) +} + +/// The order of chromosomes in the report: the number, or 99 for any +/// other name (so chrX and chrY sort together, by position). +fn chrom_key(c: &str) -> i128 { + let c = c.strip_prefix("chr").unwrap_or(c); + if pycompat::text::isdigit(c) { + pycompat::num::parse_int(c).unwrap_or(99) + } else { + 99 + } +} + +/// A merged segment: chromosome, start, end, the highest LOD, and which +/// archaic genomes it holds. +type Merged = (String, i64, i64, f64, Vec<&'static str>); + +/// Overlapping or touching segments of one class as one. +fn merge(mut segs: Vec<(Seg, &'static str)>) -> Vec { + segs.sort_by(|(a, _), (b, _)| (chrom_key(&a.0), a.1, a.2).cmp(&(chrom_key(&b.0), b.1, b.2))); + let mut out: Vec = Vec::new(); + for ((chrom, start, end, lod), who) in segs { + match out.last_mut() { + Some(m) if m.0 == chrom && start <= m.2 => { + m.2 = m.2.max(end); + m.3 = py_max(m.3, lod); + if !m.4.contains(&who) { + m.4.push(who); + } + } + _ => out.push((chrom, start, end, lod, vec![who])), + } + } + out +} + +fn overlaps(chrom: &str, s: i64, e: i64, others: &[Merged]) -> bool { + others.iter().any(|o| o.0 == chrom && o.1 < e && s < o.2) +} + +/// A row of the report: chromosome, start, end, ancestry, matched, LOD. +type Row = (String, i64, i64, &'static str, Vec<&'static str>, f64); + +/// Neanderthal: the union of the Altai and Vindija segments. Denisovan: +/// the Denisova segments that overlap no Neanderthal one (a stretch both +/// match is counted as Neanderthal, the more common source outside +/// Oceania). +fn classify(per: &[Vec; 3]) -> Vec { + let nea = merge( + NEANDERTHALS + .iter() + .enumerate() + .flat_map(|(k, n)| per[k].iter().map(move |s| (s.clone(), *n))) + .collect(), + ); + let den = merge(per[2].iter().map(|s| (s.clone(), "Denisova")).collect()); + let mut rows: Vec = Vec::new(); + for (c, s, e, l, mut who) in nea.iter().cloned() { + who.sort_unstable(); + if overlaps(&c, s, e, &den) { + who.push("Denisova"); + } + rows.push((c, s, e, "Neanderthal", who, l)); + } + for (c, s, e, l, _) in den.iter().cloned() { + if !overlaps(&c, s, e, &nea) { + rows.push((c, s, e, "Denisovan", vec!["Denisova"], l)); + } + } + rows.sort_by(|a, b| (chrom_key(&a.0), a.1).cmp(&(chrom_key(&b.0), b.1))); + rows +} + +/// Genes per chromosome: (start, end, name), sorted. +type Genes = std::collections::HashMap>; + +/// Protein-coding genes of a GENCODE gene GTF, per chromosome, sorted by +/// (start, end, name). +fn read_genes(path: &str) -> Out { + let mut genes = Genes::new(); + let mut t = Text::open(path)?; + while let Some(line) = t.next()? { + if line.starts_with('#') { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 9 || c[2] != "gene" || !c[8].contains("gene_type \"protein_coding\"") { + continue; + } + if let Some(name) = gene_name(c[8]) { + let (s, e) = (need_int(c[3])?, need_int(c[4])?); + genes + .entry(c[0].to_owned()) + .or_default() + .push((s, e, name.to_owned())); + } + } + for v in genes.values_mut() { + v.sort(); + } + Ok(genes) +} + +/// The first `gene_name "X"` with a non-empty X (the regex +/// `gene_name "([^"]+)"`). +fn gene_name(attrs: &str) -> Option<&str> { + const KEY: &str = "gene_name \""; + attrs.match_indices(KEY).find_map(|(i, _)| { + let rest = &attrs[i + KEY.len()..]; + let end = rest.find('"')?; + (end > 0).then(|| &rest[..end]) + }) +} + +/// The names of the genes overlapping `[start, end)`, once each, in order. +/// IBDmix's start is the segment's first site and its end the next site, +/// past the segment; a GTF gene is `[s, e]`, 1-based. +fn genes_in<'a>(genes: &'a Genes, chrom: &str, start: i64, end: i64) -> Vec<&'a str> { + let name = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let mut seen: Vec<&str> = Vec::new(); + for (s, e, n) in genes.get(&name).map_or(&[][..], Vec::as_slice) { + if *s >= end { + break; // sorted by start: no later gene starts before the end + } + if *e >= start && !seen.contains(&n.as_str()) { + seen.push(n); + } + } + seen +} + +fn cmd_report(a: &Args, out: &mut dyn Write) -> Out<()> { + let spans_path = a.req("--spans"); + let mut t = Text::open(spans_path)?; + let head = t.next()?.unwrap_or("").to_owned(); + if head + .split('\t') + .take(4) + .ne(["chrom", "first", "last", "sites"]) + { + return die(format!( + "{spans_path} is not the spans table of the genotypes step" + )); + } + let mut spans: Vec<(String, i64, i64, i64)> = Vec::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 4 { + return die(format!( + "{spans_path}: a row has {} columns, want 4", + c.len() + )); + } + spans.push(( + c[0].to_owned(), + need_int(c[1])?, + need_int(c[2])?, + need_int(c[3])?, + )); + } + if spans.is_empty() { + return die(format!("{spans_path} lists no chromosome")); + } + let spans_bp: i64 = spans.iter().map(|(_, f, l, _)| l - f + 1).sum(); + let ibdmix = a.req("--ibdmix-dir"); + let mut per: [Vec; 3] = [Vec::new(), Vec::new(), Vec::new()]; + for (chrom, ..) in &spans { + for (k, n) in ARCHAICS.iter().enumerate() { + let p = join(ibdmix, &format!("{chrom}.{n}.txt")); + if !Path::new(&p).is_file() { + return die(format!("no IBDmix output {p}")); + } + per[k].extend(read_segments(&p)?); + } + } + let rows = classify(&per); + let genes_arg = a.get("--genes").unwrap_or(""); + let genes = if genes_arg.is_empty() { + Default::default() + } else { + read_genes(genes_arg)? + }; + let genes_of = |c: &str, s: i64, e: i64| -> Vec<&str> { + if genes_arg.is_empty() { + Vec::new() + } else { + genes_in(&genes, c, s, e) + } + }; + let sample = a.req("--sample"); + let pop = a.get("--pop").unwrap_or("ALL"); + let out_dir = a.req("--out-dir"); + std::fs::create_dir_all(out_dir).map_err(|e| cannot_write(Path::new(out_dir), &e))?; + + let seg_name = format!("{sample}_archaic_segments.tsv"); + let mut text = format!("{SEGMENT_HEADER}\n"); + for (c, s, e, anc, who, lod) in &rows { + let g = genes_of(c, *s, *e); + text += &format!( + "{c}\t{s}\t{e}\t{}\t{anc}\t{}\t{}\t{}\n", + e - s, + who.join(","), + fixed(*lod, 1), + if g.is_empty() { + ".".to_owned() + } else { + g.join(",") + } + ); + } + put(&Path::new(out_dir).join(&seg_name), text.as_bytes())?; + + // The summary of each ancestry. + let mut summary: Vec<(String, String)> = Vec::new(); + for anc in ["Neanderthal", "Denisovan"] { + let r: Vec<&Row> = rows.iter().filter(|x| x.3 == anc).collect(); + let bp: i64 = r.iter().map(|x| x.2 - x.1).sum(); + let key = anc.to_lowercase(); + summary.push((format!("{key}_segments"), r.len().to_string())); + summary.push((format!("{key}_bp"), bp.to_string())); + summary.push(( + format!("{key}_percent"), + if spans_bp != 0 { + fixed(100.0 * bp as f64 / spans_bp as f64, 2) + } else { + "0.00".to_owned() + }, + )); + summary.push(( + format!("{key}_longest_bp"), + r.iter().map(|x| x.2 - x.1).max().unwrap_or(0).to_string(), + )); + } + let sv = |k: &str| { + summary + .iter() + .find(|(key, _)| key == k) + .map(|(_, v)| v.as_str()) + .expect("a summary key") + }; + let mut longest: Vec<&Row> = rows.iter().collect(); + longest.sort_by(|a, b| { + (-(a.2 - a.1), chrom_key(&a.0), a.1).cmp(&(-(b.2 - b.1), chrom_key(&b.0), b.1)) + }); + longest.truncate(TOP); + let sites: i64 = spans.iter().map(|s| s.3).sum(); + let mut text = String::from("key\tvalue\n"); + let meta: [(&str, String); 10] = [ + ("sample", sample.to_owned()), + ("method", "IBDmix".to_owned()), + ( + "archaic_genomes", + "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan".to_owned(), + ), + ("population", pop.to_owned()), + ("min_lod", general(MIN_LOD, 6)), + ("min_length_bp", MIN_LENGTH.to_string()), + ("analysed_bp", spans_bp.to_string()), + ("analysed_chromosomes", spans.len().to_string()), + ("analysed_sites", sites.to_string()), + ( + "genes", + if genes_arg.is_empty() { + "none (no gene annotation installed)" + } else { + "GENCODE protein-coding" + } + .to_owned(), + ), + ]; + for (k, v) in meta.iter() { + text += &format!("{k}\t{v}\n"); + } + for (k, v) in &summary { + text += &format!("{k}\t{v}\n"); + } + let genes_text = |g: &[&str]| { + if g.is_empty() { + "none".to_owned() + } else { + g.join(", ") + } + }; + for (i, (c, s, e, anc, who, _)) in longest.iter().enumerate() { + let g = genes_of(c, *s, *e); + text += &format!( + "longest_{}\t{c}:{s}-{e} {anc} {} Mb ({}); genes: {}\n", + i + 1, + fixed((e - s) as f64 / 1e6, 2), + who.join("/"), + genes_text(&g) + ); + } + put( + &Path::new(out_dir).join(format!("{sample}_archaic.tsv")), + text.as_bytes(), + )?; + let mb = |k: &str| fixed(need_int(sv(k)).unwrap_or(0) as f64 / 1e6, 1); + let mut lines = vec![ + format!( + "Archaic ancestry of {sample} (IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan)" + ), + format!( + " Analysed: {} autosomes, {} Mb, {sites} sites; modern frequencies: 1000 Genomes {pop}", + spans.len(), + fixed(spans_bp as f64 / 1e6, 1) + ), + format!( + " Neanderthal: {}% ({} Mb in {} segments)", + sv("neanderthal_percent"), + mb("neanderthal_bp"), + sv("neanderthal_segments") + ), + format!( + " Denisovan: {}% ({} Mb in {} segments)", + sv("denisovan_percent"), + mb("denisovan_bp"), + sv("denisovan_segments") + ), + ]; + if !longest.is_empty() { + lines.push(format!(" The {} longest segments:", longest.len())); + for (c, s, e, anc, who, lod) in &longest { + let g = genes_of(c, *s, *e); + lines.push(format!( + " {c}:{s}-{e} {} Mb {anc} ({}, LOD {}) genes: {}", + fixed((e - s) as f64 / 1e6, 2), + who.join("/"), + fixed(*lod, 0), + genes_text(&g) + )); + } + } + lines.push(format!(" {CAVEAT}")); + lines.push(format!(" Every segment: {seg_name}")); + let body = lines.join("\n") + "\n"; + put( + &Path::new(out_dir).join(format!("{sample}_archaic.txt")), + body.as_bytes(), + )?; + let _ = out.write_all(body.as_bytes()); + Ok(()) +} + +// --- the command line ------------------------------------------------------------- + +const USAGE: &str = "usage: og archaic {table,strict,genotypes,report} ...\n"; + +const TABLE: Spec = Spec { + verb: "table", + usage: "usage: og archaic table --chrom CHROM --modern MODERN [--archaic NAME=FILE] [--mask NAME=BED] [--exclude BED] --out OUT\n", + options: &[ + "--chrom", + "--modern", + "--archaic", + "--mask", + "--exclude", + "--out", + ], + required: &["--chrom", "--modern", "--out"], +}; +const STRICT: Spec = Spec { + verb: "strict", + usage: "usage: og archaic strict --chrom CHROM --strict STRICT --rows ROWS --out OUT\n", + options: &["--chrom", "--strict", "--rows", "--out"], + required: &["--chrom", "--strict", "--rows", "--out"], +}; +const GENOTYPES: Spec = Spec { + verb: "genotypes", + usage: "usage: og archaic genotypes --table-dir TABLE_DIR --vcf VCF [--pop POP] [--pseudo PSEUDO] --out-dir OUT_DIR\n", + options: &["--table-dir", "--vcf", "--pop", "--pseudo", "--out-dir"], + required: &["--table-dir", "--vcf", "--out-dir"], +}; +const REPORT: Spec = Spec { + verb: "report", + usage: "usage: og archaic report --sample SAMPLE --ibdmix-dir IBDMIX_DIR --spans SPANS [--genes GENES] [--pop POP] --out-dir OUT_DIR\n", + options: &[ + "--sample", + "--ibdmix-dir", + "--spans", + "--genes", + "--pop", + "--out-dir", + ], + required: &["--sample", "--ibdmix-dir", "--spans", "--out-dir"], +}; + +/// `og archaic [args]`: the exit code. Results go to `out`, +/// messages to `err`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og archaic: error: the following arguments are required: verb" + ); + return 2; + }; + let spec = match verb.as_str() { + "table" => &TABLE, + "strict" => &STRICT, + "genotypes" => &GENOTYPES, + "report" => &REPORT, + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og archaic: error: argument verb: invalid choice: {}", + repr(other) + ); + return 2; + } + }; + let a = match Args::parse(spec, &args[1..]).and_then(|a| { + if spec.verb == "genotypes" { + a.int("--pseudo", PSEUDO)?; + } + Ok(a) + }) { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", spec.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", spec.usage); + let _ = writeln!(err, "og archaic {}: error: {msg}", spec.verb); + return 2; + } + }; + let done = match spec.verb { + "table" => cmd_table(&a, err), + "strict" => cmd_strict(&a, err), + "genotypes" => cmd_genotypes(&a, err), + _ => cmd_report(&a, out), + }; + match done { + Ok(()) => 0, + Err(Stop(code, msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + code + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(r: &str, alt: Option<&str>, code: u8) -> Sample { + Sample { + r#ref: r.into(), + alt: alt.map(Into::into), + code, + } + } + + fn code(p: Picked) -> u8 { + match p { + Picked::Call(c) => c.code, + Picked::Clash => 9, + } + } + + #[test] + fn gt_codes() { + assert_eq!(gt_code("0/1"), Some(1)); + assert_eq!(gt_code("1|1"), Some(2)); + assert_eq!(gt_code("0/0"), Some(0)); + assert_eq!(gt_code("./."), None); + assert_eq!(gt_code("1/2"), None); + assert_eq!(gt_code("1"), None); + assert_eq!(gt_code("0/1/1"), None); + } + + #[test] + fn pick_call_is_symmetric_in_its_code() { + let pairs = [ + (s("A", Some("G"), 1), s("A", None, 9)), + (s("A", Some("G"), 1), s("A", Some("T"), 0)), + (s("A", Some("G"), 9), s("A", Some("G"), 2)), + (s("A", Some("G"), 0), s("A", Some("T"), 0)), + (s("A", Some("G"), 1), s("A", Some("T"), 2)), + (s("A", Some("G"), 9), s("A", Some("T"), 0)), + (s("A", Some("G"), 1), s("A", Some("G"), 2)), + ]; + for (x, y) in pairs { + assert_eq!( + code(pick_call(Some(x.clone()), y.clone())), + code(pick_call(Some(y), x)) + ); + } + assert_eq!( + pick_call(Some(s("A", Some("G"), 0)), s("A", Some("T"), 0)), + Picked::Call(s("A", None, 0)) + ); + } + + #[test] + fn a_clash_ends_its_position() { + let recs = vec![ + (4000, s("T", Some("C"), 1)), + (4000, s("T", Some("G"), 1)), + (4000, s("T", Some("C"), 1)), + (10, s("A", Some("G"), 2)), + ]; + assert_eq!( + settle(recs), + vec![(10, s("A", Some("G"), 2)), (4000, s("T", None, 9))] + ); + } + + #[test] + fn pseudo_columns_carry_the_count() { + let p = pseudo_columns(2); + assert_eq!(p, ["0\t0", "1\t0", "2\t0", "2\t1", "2\t2"]); + } + + #[test] + fn table_names() { + assert_eq!(table_chrom("chr1.tsv.gz"), Some("chr1")); + assert_eq!(table_chrom("chr22.tsv.gz"), Some("chr22")); + assert_eq!(table_chrom("chr19.tsv.gz"), Some("chr19")); + assert_eq!(table_chrom("chr23.tsv.gz"), None); + assert_eq!(table_chrom("chr0.tsv.gz"), None); + assert_eq!(table_chrom("chrX.tsv.gz"), None); + assert_eq!(table_chrom("chr01.tsv.gz"), None); + assert_eq!(table_chrom("chr1.tsv"), None); + } + + #[test] + fn frequencies_clamp_like_python() { + let afs: Vec = ["1e-05", "nan", "inf", "-0.0", "x", "1.5"] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let (_, _, _, clean) = table_row( + Some(("A", "G", &afs)), + [Some(("A", "G", "0/1")), None, None], + ) + .unwrap(); + assert_eq!(clean, ["1e-05", "nan", "1", "-0", "0", "1"]); + } + + #[test] + fn genes_touching_the_ends() { + let mut genes = std::collections::HashMap::new(); + let mut v = vec![ + (90, 100, "ENDS_AT_START".to_owned()), + (100, 120, "INSIDE".to_owned()), + (200, 210, "STARTS_AT_END".to_owned()), + (50, 99, "BEFORE".to_owned()), + ]; + v.sort(); + genes.insert("chr22".to_owned(), v); + assert_eq!( + genes_in(&genes, "22", 100, 200), + ["ENDS_AT_START", "INSIDE"] + ); + } + + #[test] + fn gene_names() { + assert_eq!(gene_name("gene_id \"G\"; gene_name \"ABC\";"), Some("ABC")); + assert_eq!(gene_name("gene_name \"\"; gene_name \"B\";"), Some("B")); + assert_eq!(gene_name("gene_name \"open"), None); + } + + #[test] + fn merge_keeps_the_highest_lod() { + let m = merge(vec![ + (("chr22".into(), 1_000_000, 1_300_000, 40.0), "Altai"), + (("chr22".into(), 1_200_000, 1_500_000, 50.0), "Vindija"), + (("chr22".into(), 1_500_000, 1_600_000, 7.0), "Altai"), + ]); + assert_eq!(m.len(), 1); + assert_eq!((m[0].1, m[0].2, m[0].3), (1_000_000, 1_600_000, 50.0)); + assert_eq!(m[0].4, ["Altai", "Vindija"]); + } +} diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 0594164a9..74febd042 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -22,6 +22,8 @@ //! A group name is ours: a set of reference populations we put together //! and named, not a nationality. +pub mod ancient; +pub mod archaic; pub mod chroms; pub mod order; pub mod panel; diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs new file mode 100644 index 000000000..559a4d71e --- /dev/null +++ b/rust/crates/og-intervals/src/chain.rs @@ -0,0 +1,376 @@ +//! UCSC chain files, for lifting single positions from one assembly to +//! another (`og ancient lift`). +//! +//! A chain file is a list of chains. Each starts with a header line, +//! `chain score tName tSize tStrand tStart tEnd qName qSize qStrand qStart +//! qEnd id`, followed by block lines `size dt dq` (the last one `size` +//! alone) and a blank line. The `t` side is the assembly a position is +//! lifted from (the source), the `q` side the one it is lifted to. Each +//! block is an ungapped stretch of `size` bases; `dt` and `dq` are the gaps +//! before the next block on each side. +//! +//! [`Chain::lift`] maps a position only when the blocks that cover it all +//! belong to one chain; a position in a gap, or covered by two chains, has +//! no single place and gives `None`. + +use og_io::{Lines, Utf8}; +use pycompat::num::{parse_int, py_isspace}; +use std::collections::HashMap; +use std::fmt; +use std::io::{self, BufRead}; + +/// Why a chain file could not be read. +#[derive(Debug)] +pub enum ChainError { + Io(io::Error), + /// A chain whose source strand is not `+`: the chain's id (or its + /// 0-based number when the header has no id) and the strand. + SourceStrand { + chain: String, + strand: String, + }, + /// A block line before any chain header, or after a chain's last block. + Orphan(String), + /// A line that cannot be used: its number (1-based) and why. + Line { + line: usize, + why: String, + }, +} + +impl fmt::Display for ChainError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ChainError::Io(e) => write!(f, "{e}"), + ChainError::SourceStrand { chain, strand } => { + write!(f, "chain {chain} has source strand {strand}") + } + ChainError::Orphan(line) => write!(f, "a block line outside a chain: {line}"), + ChainError::Line { line, why } => write!(f, "line {line}: {why}"), + } + } +} + +impl std::error::Error for ChainError {} + +impl From for ChainError { + fn from(e: io::Error) -> Self { + ChainError::Io(e) + } +} + +/// One ungapped block: `[start, end)` on the source, 0-based. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Block { + start: u32, + end: u32, + /// The chain's number in the file (0-based). + chain: u32, + /// The target sequence's name, an index into [`Chain::names`]. + name: u32, + /// The block's first base on the target, counted on its strand. + to_start: u32, + /// The target strand, an index into [`Chain::names`]. + strand: u32, + /// The target sequence's length. + to_size: u32, +} + +/// The blocks of one source sequence, sorted, with the furthest end of +/// each prefix, so a lookup stops where no earlier block can reach. +#[derive(Debug, Default)] +struct Source { + blocks: Vec, + reach: Vec, +} + +/// A chain file, indexed by source sequence. +#[derive(Debug, Default)] +pub struct Chain { + /// Target names and strands, interned. + names: Vec, + sources: HashMap, +} + +/// Where [`Chain::lift`] puts a position. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Lifted<'a> { + /// The target sequence. + pub chrom: &'a str, + /// The 1-based position on it. + pub pos: i64, + /// The target strand as the file writes it (`+` or `-`). + pub strand: &'a str, +} + +/// The header fields a block line needs, kept while the chain's blocks run. +struct Open { + source: String, + at: i64, + name: u32, + to_at: i64, + strand: u32, + to_size: i64, + chain: u32, +} + +impl Chain { + /// Reads a chain file (universal newlines; invalid UTF-8 is replaced). + /// Fields are split on whitespace, so a header with tabs reads too. + pub fn read(input: impl BufRead) -> Result { + let mut lines = Lines::new(input, Utf8::Replace); + let mut text = String::new(); + let mut out = Chain::default(); + let mut interned: HashMap = HashMap::new(); + let mut intern = |names: &mut Vec, s: &str| -> u32 { + if let Some(&i) = interned.get(s) { + return i; + } + let i = names.len() as u32; + names.push(s.to_owned()); + interned.insert(s.to_owned(), i); + i + }; + let mut cur: Option = None; + let mut chains: u32 = 0; + let mut n = 0; + while lines.read_line(&mut text)? { + n += 1; + let c: Vec<&str> = text.split(py_isspace).filter(|f| !f.is_empty()).collect(); + let bad = |why: String| ChainError::Line { line: n, why }; + if c.is_empty() { + cur = None; + continue; + } + if c[0] == "chain" { + if c.len() < 11 { + return Err(bad(format!( + "a chain header of {} fields, want 13", + c.len() + ))); + } + let number = chains; + chains += 1; + if c[4] != "+" { + return Err(ChainError::SourceStrand { + chain: c + .get(12) + .map_or_else(|| number.to_string(), |s| (*s).to_owned()), + strand: c[4].to_owned(), + }); + } + let num = |k: usize| coord(c[k], n, k + 1); + cur = Some(Open { + source: c[2].to_owned(), + at: num(5)?, + name: intern(&mut out.names, c[7]), + to_size: num(8)?, + strand: intern(&mut out.names, c[9]), + to_at: num(10)?, + chain: number, + }); + continue; + } + let Some(open) = cur.as_mut() else { + return Err(ChainError::Orphan( + pycompat::num::py_strip(&text).to_owned(), + )); + }; + let size = coord(c[0], n, 1)?; + let fit = |v: i64| { + u32::try_from(v).map_err(|_| bad(format!("position {v} does not fit 32 bits"))) + }; + let block = Block { + start: fit(open.at)?, + end: fit(open.at + size)?, + chain: open.chain, + name: open.name, + to_start: fit(open.to_at)?, + strand: open.strand, + to_size: fit(open.to_size)?, + }; + out.sources + .entry(open.source.clone()) + .or_default() + .blocks + .push(block); + if c.len() >= 3 { + open.at += size + coord(c[1], n, 2)?; + open.to_at += size + coord(c[2], n, 3)?; + } else { + cur = None; + } + } + for s in out.sources.values_mut() { + let names = &out.names; + s.blocks.sort_by(|a, b| { + (a.start, a.end, a.chain) + .cmp(&(b.start, b.end, b.chain)) + .then_with(|| names[a.name as usize].cmp(&names[b.name as usize])) + .then_with(|| a.to_start.cmp(&b.to_start)) + .then_with(|| names[a.strand as usize].cmp(&names[b.strand as usize])) + .then_with(|| a.to_size.cmp(&b.to_size)) + }); + let mut far = 0; + s.reach = s + .blocks + .iter() + .map(|b| { + far = far.max(b.end); + far + }) + .collect(); + } + Ok(out) + } + + /// How many blocks were read. + pub fn blocks(&self) -> usize { + self.sources.values().map(|s| s.blocks.len()).sum() + } + + /// The 1-based position `pos` of source sequence `chrom` on the target, + /// or `None` when no block covers it, or blocks of more than one chain + /// do. On a `-` target strand the position is mirrored: + /// `size - 1 - offset`, 0-based. + pub fn lift(&self, chrom: &str, pos: i64) -> Option> { + let src = self.sources.get(chrom)?; + let p = pos - 1; + // Blocks that start at or before p, walked back while one of them + // may still reach past p. + let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); + let mut first: Option<&Block> = None; + while j > 0 && i64::from(src.reach[j - 1]) > p { + j -= 1; + let b = &src.blocks[j]; + if i64::from(b.end) > p { + match first { + None => first = Some(b), + Some(f) if f.chain != b.chain => return None, + Some(_) => {} + } + } + } + let b = first?; + let mut q = i64::from(b.to_start) + (p - i64::from(b.start)); + let strand = self.names[b.strand as usize].as_str(); + if strand == "-" { + q = i64::from(b.to_size) - 1 - q; + } + Some(Lifted { + chrom: &self.names[b.name as usize], + pos: q + 1, + strand, + }) + } +} + +/// A whole-number field of line `line`, column `col`: Python's `int()`. +fn coord(field: &str, line: usize, col: usize) -> Result { + parse_int(field) + .ok() + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| ChainError::Line { + line, + why: format!("field {col} '{field}' is not a whole number"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The chain of tests/test_ancient_report.py: chr1 with a gap, chr2 on + /// the minus strand, chr3 covered twice. + const TEST: &str = "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1 +100 50 50 +50 + +chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2 +100 + +chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3 +100 + +chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4 +100 +"; + + fn chain() -> Chain { + Chain::read(TEST.as_bytes()).unwrap() + } + + fn lift(c: &Chain, chrom: &str, pos: i64) -> Option<(String, i64, String)> { + c.lift(chrom, pos) + .map(|l| (l.chrom.to_owned(), l.pos, l.strand.to_owned())) + } + + fn at(chrom: &str, pos: i64, strand: &str) -> Option<(String, i64, String)> { + Some((chrom.to_owned(), pos, strand.to_owned())) + } + + #[test] + fn blocks_move_by_their_offset() { + let c = chain(); + assert_eq!(c.blocks(), 5); + assert_eq!(lift(&c, "chr1", 1051), at("chr1", 2051, "+")); + assert_eq!(lift(&c, "chr1", 1160), at("chr1", 2160, "+")); + assert_eq!(lift(&c, "chr1", 1120), None, "a gap has no place"); + assert_eq!(lift(&c, "chr9", 1051), None, "an unknown sequence"); + } + + #[test] + fn block_ends_are_half_open() { + let c = chain(); + // The first block holds 0-based 1000..1100: 1-based 1001 to 1100. + assert_eq!(lift(&c, "chr1", 1001), at("chr1", 2001, "+")); + assert_eq!(lift(&c, "chr1", 1100), at("chr1", 2100, "+")); + assert_eq!(lift(&c, "chr1", 1000), None); + assert_eq!(lift(&c, "chr1", 1101), None); + assert_eq!(lift(&c, "chr1", 1151), at("chr1", 2151, "+")); + assert_eq!(lift(&c, "chr1", 1200), at("chr1", 2200, "+")); + assert_eq!(lift(&c, "chr1", 1201), None); + assert_eq!(lift(&c, "chr1", 0), None); + assert_eq!(lift(&c, "chr1", -5), None); + } + + #[test] + fn minus_strand_mirrors() { + // 0-based 510 is 310 on the reverse strand: 10000 - 1 - 310 = 9689. + assert_eq!(lift(&chain(), "chr2", 511), at("chr2", 9690, "-")); + } + + #[test] + fn two_chains_over_a_position_give_none() { + let c = chain(); + assert_eq!(lift(&c, "chr3", 171), None); + assert_eq!(lift(&c, "chr3", 121), at("chr3", 121, "+")); + assert_eq!(lift(&c, "chr3", 221), at("chr3", 5221, "+")); + } + + #[test] + fn a_long_block_far_back_still_counts() { + // A long block of chain 1 starts long before a short block of + // chain 2; a position inside both is covered twice. + let text = "chain 1 c 9000 + 0 5000 c 9000 + 0 5000 1\n5000\n\n\ + chain 1 c 9000 + 4000 4010 d 9000 + 0 10 2\n10\n"; + let c = Chain::read(text.as_bytes()).unwrap(); + assert_eq!(lift(&c, "c", 4005), None); + assert_eq!(lift(&c, "c", 3000), at("c", 3000, "+")); + } + + #[test] + fn errors() { + let e = Chain::read("chain 1 c 10 - 0 5 c 10 + 0 5 7\n5\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "chain 7 has source strand -"); + let e = Chain::read("chain 1 c 10 - 0 5 c 10 + 0 5\n5\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "chain 0 has source strand -"); + let e = Chain::read(" 5 1 1 \n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "a block line outside a chain: 5 1 1"); + // After a block line with one field the chain is over. + let e = Chain::read("chain 1 c 10 + 0 5 c 10 + 0 5 1\n5\n6\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "a block line outside a chain: 6"); + let e = Chain::read("chain 1 c 10 + x 5 c 10 + 0 5 1\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "line 1: field 6 'x' is not a whole number"); + } +} diff --git a/rust/crates/og-intervals/src/lib.rs b/rust/crates/og-intervals/src/lib.rs index 3e3104345..2a5008bd1 100644 --- a/rust/crates/og-intervals/src/lib.rs +++ b/rust/crates/og-intervals/src/lib.rs @@ -10,8 +10,11 @@ //! search, BED files such as the PAR regions read into them) and an //! overlap index with reciprocal overlap. //! - [`fasta`]: random access into a FASTA through its `.fai`. +//! - [`chain`]: UCSC chain files, to lift single positions to another +//! assembly. pub mod bed; +pub mod chain; pub mod fasta; pub mod mask; pub mod sets; diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index dd69a5991..35edc30d6 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -57,6 +57,8 @@ const USAGE: &str = "usage: og [args] og family ibd [args] og consumer build-sites|export|check [args] og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report [args] + og archaic table|strict|genotypes|report [args] + og ancient select|lift|person-geno|align|report [args] og vntyper report [args] og locityper report [args] og mocha report [args] @@ -252,6 +254,16 @@ fn main() -> ExitCode { &mut std::io::stdout().lock(), &mut std::io::stderr().lock(), )), + "archaic" => ExitCode::from(og_ancestry::archaic::cli( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), + "ancient" => ExitCode::from(og_ancestry::ancient::cli( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), "str" => match rest.first().map(String::as_str) { #[cfg(feature = "bam")] Some("motif") => ExitCode::from(og_bam::motif::cli( From f1d1aaeb9a116a35e7de7820468dedd4dc0f0971 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:45:48 +0100 Subject: [PATCH 02/86] refactor(ancestry): steps 43 and 44 and their setup run og archaic and og ancient The parity cases compare og with bin/archaic_report.py and bin/ancient_report.py on synthetic inputs. setup.sh stamps the archaic table with a format id instead of the Python's hash, and takes a table the last Python built as current. Bead: pgp-bs9u.21 --- modules/local/ancient/main.nf | 8 +- modules/local/archaic/main.nf | 8 +- scripts/43-archaic.sh | 8 +- scripts/44-ancient.sh | 4 +- scripts/setup.sh | 32 ++- tests/parity/cases.tsv | 51 +++++ tests/parity/gen/ancient-inputs.sh | 211 ++++++++++++++++++ tests/parity/gen/archaic-inputs.sh | 187 ++++++++++++++++ .../long-geno/o.geno.tmp.absent | 0 .../short-geno/o.geno.tmp.absent | 0 .../wide-geno/o.geno.tmp.absent | 0 .../bad-code/gt/chr22.gt.gz.absent | 0 .../golden/archaic-strict/unsorted/exit | 1 + .../archaic-strict/unsorted/s.tsv.absent | 0 .../golden/archaic-strict/unsorted/stderr | 1 + .../archaic-table/unsorted/t.vcf.absent | 0 tests/parity/oracle.ref | 9 + 17 files changed, 499 insertions(+), 21 deletions(-) create mode 100644 tests/parity/gen/ancient-inputs.sh create mode 100644 tests/parity/gen/archaic-inputs.sh create mode 100644 tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent create mode 100644 tests/parity/golden/archaic-strict/unsorted/exit create mode 100644 tests/parity/golden/archaic-strict/unsorted/s.tsv.absent create mode 100644 tests/parity/golden/archaic-strict/unsorted/stderr create mode 100644 tests/parity/golden/archaic-table/unsorted/t.vcf.absent diff --git a/modules/local/ancient/main.nf b/modules/local/ancient/main.nf index 839f6ace7..a5b83de3b 100644 --- a/modules/local/ancient/main.nf +++ b/modules/local/ancient/main.nf @@ -6,11 +6,11 @@ reference block over a site becomes a record with the reference base, and a site without reads has no line ANCIENT_GENO the sample as one more column of the panel's present-day - EIGENSTRAT genotypes (bin/ancient_report.py person-geno) + EIGENSTRAT genotypes (og ancient person-geno) ANCIENT_PCA smartpca: axes from the present-day West Eurasians, the sample projected (lsqproject) ANCIENT_REPORT the nearest ancient groups, the summary and the plot, on - the panel's axes (bin/ancient_report.py report) + the panel's axes (og ancient report) The panel is the folder setup.sh --ancient-panel writes (reference/ancient/aadr_). Runs for the samples with a gVCF only; @@ -78,7 +78,7 @@ process ANCIENT_GENO { script: """ - ancient_report.py person-geno --sites ${panel}/sites_grch38.tsv --genotypes ${genotypes} \\ + og ancient person-geno --sites ${panel}/sites_grch38.tsv --genotypes ${genotypes} \\ --snp ${panel}/moderns.snp --moderns-geno ${panel}/moderns.geno --out merged.geno --stats stats.tsv { cat ${panel}/moderns.ind; printf '%s\\tU\\tPerson\\n' '${meta.id}'; } > merged.ind @@ -148,7 +148,7 @@ process ANCIENT_REPORT { script: """ release=\$(awk -F'\\t' '\$1 == "release" {print \$2}' ${panel}/DONE) - ancient_report.py report --panel-evec ${panel}/panel.evec --run-evec ${evec} --ancients ${panel}/ancients.tsv \\ + og ancient report --panel-evec ${panel}/panel.evec --run-evec ${evec} --ancients ${panel}/ancients.tsv \\ --sample ${meta.id} --release "\$release" --stats ${stats} \\ --out-tsv ${meta.id}_ancient.tsv --out-info ${meta.id}_ancient_info.tsv \\ --out-txt ${meta.id}_ancient.txt --out-svg ${meta.id}_ancient_pca.svg diff --git a/modules/local/archaic/main.nf b/modules/local/archaic/main.nf index ec31864df..bf68a4653 100644 --- a/modules/local/archaic/main.nf +++ b/modules/local/archaic/main.nf @@ -4,13 +4,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('archaic' in --tools, with --archaic_data). Three processes, as in scripts/43-archaic.sh: - ARCHAIC_GENOTYPES bin/archaic_report.py genotypes: IBDmix's genotype file of + ARCHAIC_GENOTYPES og archaic genotypes: IBDmix's genotype file of each chromosome (the archaic genomes, the sample's VCF and the 1000 Genomes stand-ins of --archaic_pop), and each archaic genome's mask ARCHAIC_IBDMIX IBDmix against each archaic genome, LOD 4, the options of Chen et al. 2020, with that genome's mask - ARCHAIC_REPORT bin/archaic_report.py report: the segments of 50 kb or + ARCHAIC_REPORT og archaic report: the segments of 50 kb or more, the share, the ten longest with their genes Equivalent to: scripts/43-archaic.sh @@ -34,7 +34,7 @@ process ARCHAIC_GENOTYPES { script: """ - archaic_report.py genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt + og archaic genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -126,7 +126,7 @@ process ARCHAIC_REPORT { script: def genes_arg = genes ? "--genes ${genes}" : '' """ - archaic_report.py report --sample ${meta.id} --ibdmix-dir ${ibdmix} --spans ${ibdmix}/spans.tsv \\ + og archaic report --sample ${meta.id} --ibdmix-dir ${ibdmix} --spans ${ibdmix}/spans.tsv \\ --pop ${params.archaic_pop} ${genes_arg} --out-dir . cat <<-END_VERSIONS > versions.yml diff --git a/scripts/43-archaic.sh b/scripts/43-archaic.sh index c6bcb62a3..9a98b9392 100755 --- a/scripts/43-archaic.sh +++ b/scripts/43-archaic.sh @@ -6,14 +6,14 @@ # high-coverage archaic genomes, the Altai and Vindija 33.19 Neanderthals and # the Altai Denisovan, and calls the stretches you share with one of them. # It needs the archaic data setup.sh --archaic builds (reference/archaic/): -# 1. bin/archaic_report.py genotypes writes IBDmix's genotype file of each +# 1. og archaic genotypes writes IBDmix's genotype file of each # chromosome: the archaic genotypes, yours from the VCF, and 100 # stand-ins carrying the 1000 Genomes allele count of ARCHAIC_POP at each # site (IBDmix takes the modern frequency from the modern columns, and # one genome alone gives it none; docs/43-archaic.md says why this is # the same score); # 2. IBDmix scores your column against each archaic genome (LOD 4); -# 3. bin/archaic_report.py report keeps the segments of 50 kb or more, +# 3. og archaic report keeps the segments of 50 kb or more, # merges the two Neanderthals, and writes the share, the segments and # the ten longest with their protein-coding genes. # @@ -61,7 +61,7 @@ W=$(cpath "$WORK") echo "[1/3] Writing IBDmix's genotype files (your VCF, the archaic genomes, the stand-ins)..." run_in --cpus 1 --memory 4g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ + /pgp-bin/og archaic genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ --pop "$POP" --out-dir "${W}/gt" # One IBDmix run per chromosome and archaic genome, THREADS at a time, with the @@ -101,7 +101,7 @@ else echo " No GENCODE gene file (${GENES}): the segments are listed without their genes (setup.sh installs it)." fi run_in --cpus 1 --memory 2g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py report --sample "$SAMPLE" --ibdmix-dir "${W}/ibdmix" \ + /pgp-bin/og archaic report --sample "$SAMPLE" --ibdmix-dir "${W}/ibdmix" \ --spans "${W}/gt/spans.tsv" --pop "$POP" ${GARG[@]+"${GARG[@]}"} --out-dir "$(cpath "$OUT")" for f in "${OUT}/${SAMPLE}_archaic.tsv" "${OUT}/${SAMPLE}_archaic_segments.tsv"; do [ -s "$f" ] || { echo "ERROR: step 43 wrote no ${f}" >&2; exit 1; } diff --git a/scripts/44-ancient.sh b/scripts/44-ancient.sh index 28cc87a79..3206fa8f4 100755 --- a/scripts/44-ancient.sh +++ b/scripts/44-ancient.sh @@ -78,7 +78,7 @@ run_in --cpus 2 --memory 4g "$BCFTOOLS_IMAGE" \ echo "[2/4] Adding ${SAMPLE} to the present-day panel..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py person-geno --sites "${P}/sites_grch38.tsv" --genotypes "${W}/genotypes.tsv" \ + /pgp-bin/og ancient person-geno --sites "${P}/sites_grch38.tsv" --genotypes "${W}/genotypes.tsv" \ --snp "${P}/moderns.snp" --moderns-geno "${P}/moderns.geno" --out "${W}/merged.geno" --stats "${W}/stats.tsv" { cat "${PANEL}/moderns.ind"; printf '%s\tU\tPerson\n' "$SAMPLE"; } > "${WORK}/merged.ind" @@ -93,7 +93,7 @@ run_in --cpus "$THREADS" --memory 8g "$EIGENSOFT_IMAGE" smartpca -p "${W}/run.pa echo "[4/4] The nearest ancient groups..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py report --panel-evec "${P}/panel.evec" --run-evec "${W}/run.evec" \ + /pgp-bin/og ancient report --panel-evec "${P}/panel.evec" --run-evec "${W}/run.evec" \ --ancients "${P}/ancients.tsv" --sample "$SAMPLE" --release "$RELEASE" --stats "${W}/stats.tsv" \ --out-tsv "${O}/${SAMPLE}_ancient.tsv" --out-info "${O}/${SAMPLE}_ancient_info.tsv" \ --out-txt "${O}/${SAMPLE}_ancient.txt" --out-svg "${O}/${SAMPLE}_ancient_pca.svg" diff --git a/scripts/setup.sh b/scripts/setup.sh index 88b6c4943..326dfeddd 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -801,7 +801,7 @@ consumer_sites_pin_check() { # id, each checked against the md5 Dataverse publishes) become the panel of # step 44 in reference/ancient/aadr_: # panel.ind, poplist.txt, ancients.tsv who builds the axes (the present-day -# West Eurasians of bin/ancient_report.py's MODERN_GROUPS) and who is +# West Eurasians of og ancient's MODERN_GROUPS) and who is # projected (dated West Eurasians with 100,000 sites or more); # sites_grch38.tsv the autosomal sites on GRCh38 (UCSC's hg19ToHg38 chain, # each kept only when one of its alleles is the GRCh38 base); @@ -830,6 +830,7 @@ install_ancient_panel() { done echo "Installing the ancient populations panel, AADR ${AADR_VERSION} 1240K (step 44) in ${dir}" ensure_image EIGENSOFT_IMAGE 44 || return 1 + need_og "chooses the ancient panel's people and places its sites" || return 1 mkdir -p "$raw" for e in anno ind snp geno; do f="${raw}/aadr.${e}" @@ -868,9 +869,9 @@ install_ancient_panel() { # shellcheck disable=SC2016 # $1 to $4 belong to the inner bash if ! run_in --rw "$part" -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 4g "$PYTHON_IMAGE" \ bash -euo pipefail -c ' - python3 /pgp-bin/ancient_report.py select --anno "$1/aadr.anno" --ind "$1/aadr.ind" \ + /pgp-bin/og ancient select --anno "$1/aadr.anno" --ind "$1/aadr.ind" \ --out-ind "$2/panel.ind" --poplist "$2/poplist.txt" --ancients "$2/ancients.tsv" - python3 /pgp-bin/ancient_report.py lift --snp "$1/aadr.snp" --chain "$3" --fasta "$4" --out "$2/sites_grch38.tsv"' \ + /pgp-bin/og ancient lift --snp "$1/aadr.snp" --chain "$3" --fasta "$4" --out "$2/sites_grch38.tsv"' \ _ "$R" "$P" "$C" "$REF_FASTA_C"; then echo "[WARN] Could not choose the panel's people or place its sites (the lines above say why)." return 1 @@ -894,7 +895,7 @@ install_ancient_panel() { return 1 fi if ! run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py align --panel-evec "${P}/panel.evec" --run-evec "${P}/check.evec"; then + /pgp-bin/og ancient align --panel-evec "${P}/panel.evec" --run-evec "${P}/check.evec"; then echo "[WARN] The two smartpca runs on the same present-day people gave different axes (logs in ${part})." return 1 fi @@ -1836,6 +1837,13 @@ archaic_filterbed_sha() { return 1 } +# The version of the table og archaic table and strict write: a change to +# their bytes changes it, and every install rebuilds its table. +ARCHAIC_TABLE_FORMAT=archaic-table-v1 +# What code= held when bin/archaic_report.py built the table: the sha256 of +# its part above "# --- genotypes (step 43)", at its last version. +ARCHAIC_TABLE_PYTHON_SHA256=6bdc4c99a8d80d96179551e37022a3cd5324f650577b9fd64e27d84383e0f1d7 + # install_archaic: the table of step 43 in reference/archaic/ARCHAIC_DATA_VERSION # (ARCHAIC_DIR: another folder inside GENOME_DIR), one chrN.tsv.gz per # autosome, GRCh38: position, REF, ALT, the ALT count of the Altai, Vindija @@ -1858,6 +1866,7 @@ install_archaic() { cpath "$dir" >/dev/null || return 1 echo "Installing the archaic data ${ARCHAIC_DATA_VERSION} (step 43) in ${dir}" for v in BCFTOOLS_IMAGE PICARD_IMAGE PYTHON_IMAGE IBDMIX_IMAGE; do ensure_image "$v" 43 || return 1; done + need_og "builds the archaic table" || return 1 [ -s "$REF_DICT" ] || { echo "[WARN] ${REF_DICT} is missing: run $0 ${GENOME_DIR} first (it writes the sequence dictionary)"; return 1; } mkdir -p "${dir}/md5" "${dir}/download" "${dir}/work" "${dir}/mask" "${dir}/exclude" for v in "Altai|${ARCHAIC_ALTAI_MD5SUMS_SHA256}" "Vindija33.19|${ARCHAIC_VINDIJA_MD5SUMS_SHA256}" "Denisova|${ARCHAIC_DENISOVA_MD5SUMS_SHA256}"; do @@ -1893,9 +1902,18 @@ install_archaic() { chroms=$(seq 1 22 | tr '\n' ' ') fi [ -n "${chroms// /}" ] || { echo "[WARN] ARCHAIC_REGIONS=${regions} names no autosome (1 to 22)"; return 1; } - stamp="data=${ARCHAIC_DATA_VERSION} regions=${rstamp} bcftools=${BCFTOOLS_IMAGE} picard=${PICARD_IMAGE} masks=$(_digest sha256 <(printf '%s\n%s\n%s\n%s\n%s\n' "$ARCHAIC_STRICT_MASK_SHA256" "$ARCHAIC_SEGDUPS_SHA256" "$ARCHAIC_FILTERBED_SHA256_ALTAI" "$ARCHAIC_FILTERBED_SHA256_VINDIJA" "$ARCHAIC_FILTERBED_SHA256_DENISOVA")) code=$(_digest sha256 <(sed '/^# --- genotypes (step 43)/,$d' "${PGP_ROOT}/bin/archaic_report.py"))" + stamp="data=${ARCHAIC_DATA_VERSION} regions=${rstamp} bcftools=${BCFTOOLS_IMAGE} picard=${PICARD_IMAGE} masks=$(_digest sha256 <(printf '%s\n%s\n%s\n%s\n%s\n' "$ARCHAIC_STRICT_MASK_SHA256" "$ARCHAIC_SEGDUPS_SHA256" "$ARCHAIC_FILTERBED_SHA256_ALTAI" "$ARCHAIC_FILTERBED_SHA256_VINDIJA" "$ARCHAIC_FILTERBED_SHA256_DENISOVA")) code=${ARCHAIC_TABLE_FORMAT}" + # Until og archaic replaced bin/archaic_report.py, code= held the sha256 of + # that file's table half. The last Python wrote the same table bytes as + # og archaic table and strict (tests/parity), so a table it built counts as + # built here: its stamp is accepted once and rewritten. Remove this one + # release after 1.0.0. + local legacy="${stamp% code=*} code=${ARCHAIC_TABLE_PYTHON_SHA256}" local have=1 for c in $chroms; do [ -s "${dir}/chr${c}.tsv.gz" ] || have=0; done + for v in "${dir}/DONE" "${dir}"/chr*.stamp; do + [ "$(cat "$v" 2>/dev/null)" = "$legacy" ] && echo "$stamp" > "$v" + done if [ "$have" = 1 ] && [ "$(cat "${dir}/DONE" 2>/dev/null)" = "$stamp" ]; then echo "[OK] archaic data ${ARCHAIC_DATA_VERSION} (step 43): ${dir}" return 0 @@ -1975,7 +1993,7 @@ install_archaic() { echo "[WARN] Could not read the archaic genotypes of chr${c}"; return 1 fi if ! run_in --rw "$dir" -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py table --chrom "$c" --modern "$(cpath "${wk}/modern.tsv")" \ + /pgp-bin/og archaic table --chrom "$c" --modern "$(cpath "${wk}/modern.tsv")" \ --archaic "Altai=$(cpath "${wk}/Altai.tsv")" --archaic "Vindija=$(cpath "${wk}/Vindija33.19.tsv")" \ --archaic "Denisova=$(cpath "${wk}/Denisova.tsv")" \ --mask "Altai=$(cpath "${dir}/mask/Altai.chr${c}.bed.gz")" \ @@ -2009,7 +2027,7 @@ install_archaic() { echo "[WARN] the GRCh38 strict mask has no interval on chr${c}: ${dir}/strict_GRCh38.bed"; return 1 fi if ! run_in --rw "$dir" -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py strict --chrom "$c" --strict "$(cpath "${wk}/strict.bed")" \ + /pgp-bin/og archaic strict --chrom "$c" --strict "$(cpath "${wk}/strict.bed")" \ --rows "$(cpath "${wk}/lifted.tsv")" --out "$(cpath "${wk}/marked.tsv")"; then echo "[WARN] Could not mark the strict mask on chr${c}"; return 1 fi diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 586dfda38..6ec8e03fa 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -917,3 +917,54 @@ family-ibd mapbp gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree mapbp/pedigree.ts family-ibd nomap gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree nomap/pedigree.tsv --chroms nomap/chroms.txt --maps nomap/maps --phased nomap/phased --ibd nomap/ibd --out-dir out - content PYTHON_IMAGE family-ibd short gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree short/pedigree.tsv --chroms short/chroms.txt --maps short/maps --phased short/phased --ibd short/ibd --out-dir out - content PYTHON_IMAGE family-ibd twice gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree twice/pedigree.tsv --chroms twice/chroms.txt --maps twice/maps --phased twice/phased --ibd twice/ibd --out-dir out - content PYTHON_IMAGE +# Step 43, og archaic (bin/archaic_report.py) +archaic-table masked gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=altai.bed --mask Vindija=vindija.bed --exclude excl.bed --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table plain gen:archaic-inputs.sh --chrom chr22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table gz gen:archaic-inputs.sh --chrom 22 --modern modern.tsv.gz --archaic Altai=altai.tsv.gz --archaic Vindija=vindija.tsv.gz --archaic Denisova=denisova.tsv.gz --mask Denisova=altai.bed --exclude excl.bed --exclude vindija.bed --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table unsorted gen:archaic-inputs.sh --chrom 22 --modern unsorted.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table order gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Vindija=vindija.tsv --archaic Altai=altai.tsv --archaic Denisova=denisova.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table bad-name gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Neanderthal=altai.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table mask-twice gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=altai.bed --mask Altai=vindija.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-table empty-mask gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=empty.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-table empty-exclude gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --exclude empty.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-strict plain gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows.tsv --out s.tsv s.tsv bytes PYTHON_IMAGE +archaic-strict unsorted gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows-unsorted.tsv --out s.tsv - bytes PYTHON_IMAGE +archaic-strict empty-mask gen:archaic-inputs.sh --chrom 22 --strict empty.bed --rows rows.tsv --out s.tsv - bytes PYTHON_IMAGE +archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr22.Altai.mask.bed *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes default gen:archaic-inputs.sh --table-dir . --vcf s.vcf.gz --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes afr-one gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop AFR --pseudo 1 --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes two-samples gen:archaic-inputs.sh --table-dir . --vcf two.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes comes-back gen:archaic-inputs.sh --table-dir . --vcf back.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes no-table-chrom gen:archaic-inputs.sh --table-dir . --vcf none.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes bad-code gen:archaic-inputs.sh --table-dir badcode --vcf s.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes bad-pop gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop XYZ --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes bad-pseudo gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pseudo 0 --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes no-table gen:archaic-inputs.sh --table-dir emptydir --vcf s.vcf --out-dir gt - bytes PYTHON_IMAGE +archaic-report genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --genes genes.gtf --pop EUR --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE +archaic-report no-genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE +archaic-report missing-output gen:archaic-inputs.sh --sample S --ibdmix-dir rep --spans spans.tsv --out-dir rep - bytes PYTHON_IMAGE +archaic-report not-spans gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans genes.gtf --out-dir rep - bytes PYTHON_IMAGE +# Step 44, og ancient (bin/ancient_report.py) +ancient-select plain gen:ancient-inputs.sh --anno aadr.anno --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv o.ind,pops.txt,anc.tsv bytes PYTHON_IMAGE +ancient-select min-snps gen:ancient-inputs.sh --anno aadr.anno --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv --min-snps 140000 o.ind,pops.txt,anc.tsv bytes PYTHON_IMAGE +ancient-select too-few gen:ancient-inputs.sh --anno aadr.anno --ind few.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv - bytes PYTHON_IMAGE +ancient-select no-column gen:ancient-inputs.sh --anno aadr.ind --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv - bytes PYTHON_IMAGE +ancient-lift plain gen:ancient-inputs.sh --snp aadr.snp --chain hg19.chain --fasta hg38.fa --out sites.out.tsv sites.out.tsv bytes PYTHON_IMAGE +ancient-lift gz-chain gen:ancient-inputs.sh --snp aadr.snp --chain hg19.chain.gz --fasta hg38.fa --out sites.out.tsv sites.out.tsv bytes PYTHON_IMAGE +ancient-lift minus-source gen:ancient-inputs.sh --snp aadr.snp --chain minus-source.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-lift orphan-block gen:ancient-inputs.sh --snp aadr.snp --chain orphan.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-lift nothing-placed gen:ancient-inputs.sh --snp panel.geno --chain hg19.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-person-geno plain gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno panel.geno --out o.geno --stats o.stats o.geno,o.stats bytes PYTHON_IMAGE +ancient-person-geno short-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno short.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno long-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno long.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno wide-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno wide.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno none-used gen:ancient-inputs.sh --sites sites.tsv --genotypes nowhere.tsv --snp panel.snp --moderns-geno panel.geno --out o.geno --stats o.stats o.geno,o.stats bytes PYTHON_IMAGE +ancient-align flipped gen:ancient-inputs.sh --panel-evec panel.evec --run-evec flipped.evec - bytes PYTHON_IMAGE +ancient-align five-pcs gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --pcs 5 - bytes PYTHON_IMAGE +ancient-align differ gen:ancient-inputs.sh --panel-evec panel.evec --run-evec bad.evec - bytes PYTHON_IMAGE +ancient-align too-few gen:ancient-inputs.sh --panel-evec panel.evec --run-evec few.evec - bytes PYTHON_IMAGE +ancient-align not-evec gen:ancient-inputs.sh --panel-evec stats.tsv --run-evec run.evec - bytes PYTHON_IMAGE +ancient-report plain gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S1 --release v66.p1 --stats stats.tsv --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg o.tsv,o.info,o.txt,o.svg bytes PYTHON_IMAGE +ancient-report no-stats gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S1 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg o.tsv,o.info,o.txt,o.svg bytes PYTHON_IMAGE +ancient-report not-person gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S2 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg - bytes PYTHON_IMAGE +ancient-report axes-differ gen:ancient-inputs.sh --panel-evec panel.evec --run-evec bad.evec --ancients ancients.tsv --sample S1 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg - bytes PYTHON_IMAGE diff --git a/tests/parity/gen/ancient-inputs.sh b/tests/parity/gen/ancient-inputs.sh new file mode 100644 index 000000000..74d6a9a35 --- /dev/null +++ b/tests/parity/gen/ancient-inputs.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# The inputs of the ancient cases of step 44 (ancient-select, ancient-lift, +# ancient-person-geno, ancient-align, ancient-report), all synthetic, +# written with awk: +# aadr.anno, aadr.ind, few.ind +# an AADR-style annotation (long header cells, a quoted one, +# ASSESSMENT WARNINGS before ASSESSMENT) with present-day people +# of the reference groups, one individual under two genetic +# ids, ancient people inside and outside West Eurasia, below the +# site floor, undated, failing the assessment; the .ind file +# (unknown ids, short lines) and one that matches too few +# hg19.chain, hg19.chain.gz, minus-source.chain, orphan.chain, hg38.fa(.fai), stale.fa(.fai) +# a chain with a gap, a minus-strand block, two chains over one +# stretch and a block that moves to another chromosome; a +# reference with lower-case lines; one whose .fai is wrong +# aadr.snp sites in blocks, in gaps, on block edges, on the minus +# strand, off the autosomes, with equal or non-base alleles and +# with a fractional position +# sites.tsv, genotypes.tsv, panel.snp, panel.geno, short.geno, +# long.geno, wide.geno, nowhere.tsv +# person-geno: two SNPs at one position, other alleles, missing +# and haploid calls; a .geno that is short, long or uneven +# panel.evec, run.evec, flipped.evec, bad.evec, few.evec, ancients.tsv, stats.tsv +# two smartpca runs on one panel (the second with PC2's sign +# flipped), one whose axes differ, one with too few present-day +# people, and the ancient people's table (names to escape, a +# long name, undated people, dates on both sides of year 1) +set -euo pipefail + +# --- select ------------------------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + print "Genetic ID", "Master ID", "Individual ID (PubMed)", \ + "Date mean in BP in years before 1950 CE [OxCal mu for a direct radiocarbon date]", \ + "Date standard deviation in BP [OxCal sigma]", "Full Date One of two formats", \ + "Group ID", "\"Locality\"", "Political Entity", "Latitude", "Longitude", "Data type", \ + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", \ + "ASSESSMENT WARNINGS (Xcontam interval is listed if lower bound is >0.005)", " ASSESSMENT " > "aadr.anno" + split("CEU French Sardinian Basque GBR IBS TSI Druze Armenian Polish Mozabite Saami", M, " ") + for (i = 1; i <= 90; i++) { + g = M[i % 12 + 1] + a = (i % 9 == 0) ? "QUESTIONABLE" : (i % 10 == 0 ? " pass (low coverage)" : "PASS") + dt = (i % 15 == 0) ? "1240K" : "Shotgun.diploid" + print "M" i, "M" i, "Ind_M" i, (i % 17 == 0 ? "-0.0" : "0"), "0", "present", g, "Paris\tFrance", "48.8", "2.3", dt, 900000 + i, "", a >> "aadr.anno" + } + print "M91", "M91", "Ind_M5", "0", "0", "present", "CEU", "Utah", "USA", "40", "-111", "Shotgun.diploid", 999999999, "", "PASS" >> "aadr.anno" + split("Germany_CordedWare England_BellBeaker Turkey_N Italy_Imperial_Rome Hungary_LBA_with_a_very_long_name_that_runs_past_45 Türkiye_Neolithic", G, " ") + for (i = 1; i <= 60; i++) { + g = G[i % 7 + 1] + bp = (i % 13 == 0) ? ".." : 500 + 137 * i + lat = (i % 11 == 0) ? 10 : 30 + i % 40 + lon = (i % 19 == 0) ? "" : -20 + i + snps = (i % 8 == 0) ? 99999 : 100000 + 1000 * i + a = (i % 14 == 0) ? "FAIL" : (i % 6 == 0 ? "PROVISIONAL_PASS" : "MERGE_PASS") + ind = (i % 10 == 0) ? "" : "Ind_A" int(i / 2) + print "A" i, "A" i, ind, bp, "50", " 2500-2000 BCE ", g, "Site " (i % 3), (i % 4 == 0 ? "" : "Country" (i % 2)), lat, lon, "1240K", snps, "x", a >> "aadr.anno" + } + print "short", "row" >> "aadr.anno" + for (i = 1; i <= 91; i++) print "M" i, (i % 2 ? "M" : "F"), "Pop" > "aadr.ind" + for (i = 1; i <= 60; i++) print " A" i, "U", "Pop" >> "aadr.ind" + print "X1 M" >> "aadr.ind" + print "Unknown", "M", "Pop" >> "aadr.ind" + for (i = 1; i <= 20; i++) print "M" i, "M", "Pop" > "few.ind" +}' + +# --- lift --------------------------------------------------------------------------- +cat > hg19.chain <<'EOF' +chain 1000 chr1 100000 + 1000 1600 chr1 3000 + 2000 2600 1 +200 100 100 +300 + +chain 900 chr2 50000 + 500 900 chr2 3000 - 100 500 2 +400 + +chain 800 chr3 50000 + 100 400 chr3 3000 + 100 400 3 +300 + +chain 700 chr3 50000 + 300 500 chr3 3000 + 1300 1500 4 +200 + +chain 600 chr4 50000 + 0 500 chr5 3000 + 0 500 5 +500 + +chain 500 chr5 50000 + 100 1100 chr5 3000 + 1100 2100 6 +1000 +EOF +gzip -n -c hg19.chain > hg19.chain.gz +printf 'chain 1 chr1 100 - 0 10 chr1 100 + 0 10 9\n10\n' > minus-source.chain +printf 'chain 1 chr1 100 + 0 10 chr1 100 + 0 10 9\n10\n5\n' > orphan.chain +awk 'BEGIN { + s = "ACGTTGCAGC"; off = 0 + for (c = 1; c <= 5; c++) { + h = ">chr" c; printf "%s\n", h > "hg38.fa"; off += length(h) + 1 + printf "chr%d\t3000\t%d\t60\t61\n", c, off > "hg38.fa.fai" + printf "chr%d\t3000\t%d\t60\t61\n", c, off + 1 > "stale.fa.fai" + line = "" + for (i = 0; i < 3000; i++) { + line = line substr(s, (i * 7 + c * 3) % 10 + 1, 1) + if (length(line) == 60) { + if (int(i / 60) % 3 == 1) line = tolower(line) + printf "%s\n", line > "hg38.fa"; off += 61; line = "" + } + } + } +}' +cp hg38.fa stale.fa +awk 'BEGIN { + OFS = " " + split("A C A G A T C G C T G T", P, " ") + n = 0 + pos[1] = "999 1000 1001 1100 1200 1201 1250 1300 1301 1450 1600 1601" + pos[2] = "500 501 600 750 900 901" + pos[3] = "100 101 250 300 301 350 400 401 450 500 501" + pos[4] = "1 250 500" + pos[5] = "101 500 1100 1101" + for (c = 1; c <= 5; c++) { + k = split(pos[c], Q, " ") + for (j = 1; j <= k; j++) { + for (p = 1; p <= 6; p++) { + n++ + a1 = P[2 * p - 1]; a2 = P[2 * p] + if (n % 5 == 0) { t = a1; a1 = a2; a2 = t } + if (n % 23 == 0) { a1 = tolower(a1) } + q = (n % 29 == 0) ? Q[j] ".0" : Q[j] + printf " rs%d\t%d 0.0123 %s %s %s\n", n, c, q, a1, a2 > "aadr.snp" + } + } + } + print "rsA", 1, "0", 1150, "A", "A" > "aadr.snp" + print "rsN", 1, "0", 1150, "N", "G" > "aadr.snp" + print "rsX", 23, "0", 1150, "A", "G" > "aadr.snp" + print "rsY", "X", "0", 1150, "A", "G" > "aadr.snp" + print "rs0", 0, "0", 1150, "A", "G" > "aadr.snp" + print "short", 1, "0", 1150, "A" > "aadr.snp" + print "rsDup", 1, "0", 1150, "A", "G" > "aadr.snp" + print "rsDup0", 1, "0", 1150, "A", "G" > "aadr.snp" +}' + +# --- person-geno -------------------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("A G C T A C G T", P, " ") + print "snp", "chrom", "pos", "allele1", "allele2" > "sites.tsv" + for (i = 1; i <= 200; i++) { + pos = (i == 51) ? 500 : 10 * i + print "rs" i, "chr1", pos, P[i % 4 * 2 + 1], P[i % 4 * 2 + 2] > "sites.tsv" + } + print "rs999", "chr1" > "sites.tsv" + split("A/G A|A G/G ./. C/T A a/g G|A A/G/G", T, " ") + for (i = 1; i <= 220; i += 3) { + a1 = P[i % 4 * 2 + 1]; a2 = P[i % 4 * 2 + 2] + g = T[i % 10 + 1] + gsub(/A/, "1", g); gsub(/G/, "2", g); gsub(/a/, "3", g); gsub(/g/, "4", g) + gsub(/1/, a1, g); gsub(/2/, a2, g); gsub(/3/, tolower(a1), g); gsub(/4/, tolower(a2), g) + print "chr1", 10 * i, a1, a2, g > "genotypes.tsv" + } + print "chr1", 500, "A", "G", "A/G" > "genotypes.tsv" + print "chr1", 510, "C", "T", "T/T" > "genotypes.tsv" + print "chr1", 510, "C", "T", "C/T" > "genotypes.tsv" + print "1", 20, "A", "G", "A/G" > "genotypes.tsv" + print "chr1", 30 > "genotypes.tsv" + print "chr2", 10, "A", "G", "A/G" > "nowhere.tsv" + for (i = 1; i <= 230; i++) { + if (i == 100) print "" > "panel.snp" + print "rs" i, 1, "0.0", 10 * i, "A", "G" > "panel.snp" + g = sprintf("%d%d%d%d%d", i % 3, (i + 1) % 3, 9, i % 2, 2) + print g > "panel.geno" + if (i < 230) print g > "short.geno" + print g > "long.geno" + print (i == 77 ? g "0" : g) > "wide.geno" + } + print "" > "long.geno" +}' + +# --- align and report --------------------------------------------------------------- +awk 'BEGIN { + split("French Sardinian CEU Basque GBR", M, " ") + printf " #eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "panel.evec" + printf "#eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "run.evec" + printf "#eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "bad.evec" + printf "#eigvals: 5.512 3.201\n" > "few.evec" + for (i = 1; i <= 40; i++) { + p1 = sin(i * 1.7) * 0.05; p2 = cos(i * 0.9) * 0.04; p3 = sin(i * 0.31) * 0.03; p4 = cos(i * 2.3) * 0.02; p5 = sin(i) * 0.01 + lab = M[i % 5 + 1] + printf "%12s %10.6f %10.6f %10.6f %10.6f %10.6f %15s\n", "M" i, p1, p2, p3, p4, p5, lab > "panel.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M" i, p1, -p2, p3, -p4, p5, lab > "run.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M" i, (i % 2 ? p1 : -p1), p2, p3, p4, p5, lab > "bad.evec" + if (i <= 9) printf "%s %.6f %.6f %s\n", "M" i, p1, p2, lab > "few.evec" + } + for (i = 1; i <= 60; i++) { + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "A" i, sin(i * 0.77) * 0.06, cos(i * 1.31) * 0.05, sin(i * 0.5) * 0.02, cos(i) * 0.02, 0.001, "Ancient" > "panel.evec" + } + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M3", 0.01, 0.02, 0.03, 0.04, 0.05, "Other" > "panel.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S1", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Person" > "run.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S1", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Person" > "bad.evec" + printf "%s %.6f %.6f %s\n", "S1", 0.0123, -0.0045, "Person" > "few.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S2", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Ancient" > "run.evec" + OFS = "\t" + print "id", "group", "date_bp", "date_sd", "full_date", "locality", "country", "lat", "lon", "snps" > "ancients.tsv" + split("Germany_CordedWare England_BellBeaker Turkey_N Italy_Imperial_Rome Hungary_LBA_with_a_very_long_name_that_runs_past_45 Türkiye_Neolithic", G, " ") + split("1650 1950 1949 2450.5 4250 7300 9960 6000 3000 .", D, " ") + for (i = 1; i <= 60; i++) { + loc = (i % 3 == 0) ? "." : "Site " (i % 4) + print "A" i, G[i % 7 + 1], D[i % 10 + 1], "50", ".", loc, "Country" (i % 2), "45", "10", 200000 > "ancients.tsv" + } + print "A99", "Nowhere", "5000", "50", ".", ".", ".", "45", "10", 200000 > "ancients.tsv" + print "A7", "Germany_CordedWare", "5100", "50", ".", "Halle", "Germany" > "ancients.tsv" + print "" > "ancients.tsv" + printf "panel_sites\t230\nsites_used\t61\nother\n" > "stats.tsv" +}' +cp run.evec flipped.evec diff --git a/tests/parity/gen/archaic-inputs.sh b/tests/parity/gen/archaic-inputs.sh new file mode 100644 index 000000000..5d179c532 --- /dev/null +++ b/tests/parity/gen/archaic-inputs.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# The inputs of the archaic cases of step 43 (archaic-table, archaic-strict, +# archaic-genotypes, archaic-report), all synthetic, written with awk: +# modern.tsv, altai.tsv, vindija.tsv, denisova.tsv +# bcftools query lines of one GRCh37 chromosome (22): 1000 Genomes +# SNVs with their six frequencies (indels, a split multi-allelic +# site, frequencies that need :.6g's exponent form, NaN, out of +# range and unreadable values) and the three genomes' calls +# (missing, partial, multi-allelic, gVCF <*> and REF-mismatch +# calls, and sites only an archaic genome carries), also gzipped +# altai.bed, vindija.bed, excl.bed, empty.bed +# the masks: chromosome 22 named with and without chr, unsorted, +# overlapping, reversed and empty intervals, track lines +# unsorted.tsv a modern file that goes back +# rows.tsv, rows-unsorted.tsv, strict.bed +# lifted GRCh38 rows (repeated positions) and the strict mask +# chr21.tsv.gz, chr22.tsv.gz, chrX.tsv.gz +# the GRCh38 table step 43 reads (chrX is not a table name), +# with masked codes and frequencies that round half to even +# s.vcf, s.vcf.gz, two.vcf, back.vcf, none.vcf, badcode/ +# the sample's VCF: split records, clashes, indels, a +# multi-allelic record, RefCall, missing GT, a chromosome the +# table lacks; a two-sample VCF, one whose chromosome comes +# back, one with no table chromosome +# chr21.Altai.txt ... chr22.Denisova.txt, spans.tsv, genes.gtf +# IBDmix's segments (short, weak, overlapping and touching +# ones, other IDs) and a GENCODE-style gene GTF +set -euo pipefail + +# --- table (GRCh37 inputs) ---------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0.5 1e-05 0.000123 . nan 1.5 -0.1 0.123456789 1 0 2.5e-7 x", F, " ") + for (i = 1; i <= 60; i++) { + pos = 1000 + 10 * i + r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + if (i % 11 == 0) { print pos, r "T", a, F[1], F[2], F[3], F[4], F[5], F[6] > "modern.tsv"; continue } + if (i % 13 == 0) { + print pos, r, a, "0.1", "0.2", "0.3", "0.4", "0.5", "0.6" > "modern.tsv" + print pos, r, B[(i + 2) % 4 + 1], "0.1", "0.2", "0.3", "0.4", "0.5", "0.6" > "modern.tsv" + continue + } + if (i % 7 == 3) continue + print pos, r, a, F[i % 12 + 1], F[(i + 1) % 12 + 1], F[(i + 2) % 12 + 1], F[(i + 3) % 12 + 1], F[(i + 4) % 12 + 1], F[(i + 5) % 12 + 1] > "modern.tsv" + } + split("0/0 0/1 1/1 ./. 1/2 0|1 1 0/1/1 . 1|1", G, " ") + for (i = 1; i <= 64; i++) { + pos = 1000 + 10 * i + (i % 9 == 0 ? 5 : 0) + r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + if (i % 3 != 0) print pos, r, (i % 10 == 4 ? "<*>" : a), G[i % 10 + 1] > "altai.tsv" + if (i % 4 != 1) print pos, (i % 17 == 0 ? "C" : r), (i % 6 == 0 ? "." : a), G[(i + 3) % 10 + 1] > "vindija.tsv" + if (i % 5 != 2) print pos, r, (i % 8 == 0 ? B[(i + 2) % 4 + 1] : a), G[(i + 7) % 10 + 1] > "denisova.tsv" + } +}' +for f in modern altai vindija denisova; do gzip -n -c "${f}.tsv" > "${f}.tsv.gz"; done +printf '1200\tA\tG\t0.1\t0.1\t0.1\t0.1\t0.1\t0.1\n1100\tA\tG\t0.1\t0.1\t0.1\t0.1\t0.1\t0.1\n' > unsorted.tsv +cat > altai.bed <<'EOF' +track name=altai +# a comment +22 1300 1500 +22 1000 1100 +22 1080 1250 +22 1600 1600 +22 1700 1650 +chr21 0 100000 +EOF +printf 'browser position chr22\nchr22\t1000\t1400\nchr22\t1450\t1700\n' > vindija.bed +printf 'chr22\t1195\t1215\n22\t1500\t1520\n' > excl.bed +printf 'chr21\t0\t1000\n' > empty.bed + +# --- strict (lifted rows) ----------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("0 1 2 9 10 11 12", C, " ") + for (i = 1; i <= 40; i++) { + pos = 5000 + 7 * i + row = "chr22" OFS pos OFS "A" OFS "G" OFS C[i % 7 + 1] OFS C[(i + 2) % 7 + 1] OFS C[(i + 4) % 7 + 1] OFS "0.1" OFS "0.2" OFS "0.3" OFS "0.4" OFS "0.5" OFS "0.6" + print row > "rows.tsv" + if (i % 9 == 0) print "chr22" OFS pos OFS "C" OFS "T" OFS "1" OFS "1" OFS "1" OFS "0" OFS "0" OFS "0" OFS "0" OFS "0" OFS "0" > "rows.tsv" + } +}' +{ head -5 rows.tsv; printf 'chr22\t5001\tA\tG\t1\t1\t1\t0\t0\t0\t0\t0\t0\n'; tail -3 rows.tsv; } > rows-unsorted.tsv +printf 'chr22\t5010\t5100\n22\t5150\t5200\nchr22\t5240\t5260\n' > strict.bed + +# --- genotypes (the GRCh38 table and the sample) ------------------------------------ +HEAD='chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS' +table() { # CHROM START N + awk -v c="$1" -v s="$2" -v n="$3" 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0 1 2 9 10 11 12", C, " ") + split("0.0025 0.0075 0.5 1e-05 . nan 0.99 1.2 -0.3 0.0125 0.333333", F, " ") + for (i = 1; i <= n; i++) { + pos = s + 10 * i + print c, pos, B[i % 4 + 1], B[(i + 1) % 4 + 1], C[i % 7 + 1], C[(i + 3) % 7 + 1], C[(i * 5) % 7 + 1], \ + F[i % 11 + 1], F[(i + 1) % 11 + 1], F[(i + 2) % 11 + 1], F[(i + 3) % 11 + 1], F[(i + 4) % 11 + 1], F[(i + 5) % 11 + 1] + } + }' +} +{ printf '%b\n' "$HEAD"; table chr21 1000 30; } | gzip -n > chr21.tsv.gz +{ printf '%b\n' "$HEAD"; table chr22 2000 50; } | gzip -n > chr22.tsv.gz +{ printf '%b\n' "$HEAD"; table chrX 3000 5; } | gzip -n > chrX.tsv.gz +mkdir -p badcode emptydir +{ printf '%b\n' "$HEAD"; table chr22 2000 4 | awk -F'\t' -v OFS='\t' 'NR == 3 { $5 = 3 } { print }'; } | gzip -n > badcode/chr22.tsv.gz +vcf_head() { + printf '##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$1" +} +{ + vcf_head S + awk 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0/1 1/1 0/0 ./. 0|1 1|1 0/2 1", G, " ") + print "chr1", 50, ".", "A", "G", ".", "PASS", ".", "GT", "0/1" + for (i = 1; i <= 30; i += 2) { + pos = 1000 + 10 * i; r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + print "chr21", pos, ".", r, a, ".", (i % 5 == 0 ? "RefCall" : "PASS"), ".", "GT:GQ", G[i % 8 + 1] ":30" + } + for (i = 1; i <= 50; i++) { + pos = 2000 + 10 * i; r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1]; o = B[(i + 2) % 4 + 1] + k = i % 12 + if (k == 0) print "chr22", pos, ".", r, a, ".", ".", ".", "GT", G[i % 8 + 1] + else if (k == 1) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/0" } + else if (k == 2) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/0"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/0" } + else if (k == 3) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "1/1" } + else if (k == 4) { print "chr22", pos, ".", r, a "," o, ".", "PASS", ".", "GT", "1/2" } + else if (k == 5) { print "chr22", pos, ".", r "T", r, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "1/1" } + else if (k == 6) { print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "1/1" } + else if (k == 7) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GQ", "20" } + else if (k == 8) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT:DP", "./.:3" } + else if (k == 9) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "./." } + else if (k == 10) { print "chr22", pos, ".", r, ".", ".", "PASS", ".", "GT", "0/0" } + else print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT:AD", "1/1:0,9" + } + print "chr22", 9999, ".", "A", "G", ".", "PASS", ".", "GT", "0/1" + }' +} > s.vcf +gzip -n -c s.vcf > s.vcf.gz +{ printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n'; printf 'chr22\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\t0/0\n'; } > two.vcf +{ vcf_head S; printf 'chr22\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchr21\t1010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t2020\t.\tG\tT\t.\tPASS\t.\tGT\t0/1\n'; } > back.vcf +{ vcf_head S; printf 'chr1\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchrX\t3010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\n'; } > none.vcf + +# --- report (IBDmix output) --------------------------------------------------------- +printf 'chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\nchr21\t1000000\t46000000\t3000\t400\t2\nchr22\t16000000\t50800000\t2500\t300\t1\n' > spans.tsv +seg() { printf 'ID\tchrom\tstart\tend\tslod\n'; cat; } +seg > chr21.Altai.txt <<'EOF' +SAMPLE 21 1000000 1200000 12.5 +SAMPLE 21 1150000 1400000 7.25 +SAMPLE 21 2000000 2040000 30 +SAMPLE 21 3000000 3100000 3.99 +P001 21 5000000 9000000 99 +SAMPLE 21 9000000 9500000 4 +EOF +seg > chr21.Vindija.txt <<'EOF' +SAMPLE 21 1400000 1600000 8.5 +SAMPLE 21 9000000 9300000 45.5 +SAMPLE chr21 20000000 20060000 4.5 +EOF +seg > chr21.Denisova.txt <<'EOF' +SAMPLE 21 1100000 1300000 6 +SAMPLE 21 30000000 30500000 14.5 +SAMPLE 21 30400000 30700000 5.5 +EOF +seg > chr22.Altai.txt <<'EOF' +SAMPLE 22 17000000 17800000 22.5 +SAMPLE 22 40000000 40050000 9 +EOF +seg > chr22.Vindija.txt <<'EOF' +SAMPLE 22 17500000 18100000 10.5 +EOF +seg > chr22.Denisova.txt <<'EOF' +SAMPLE 22 25000000 25900000 8.5 +SAMPLE 22 17000000 17050000 15 +EOF +cat > genes.gtf <<'EOF' +##description: synthetic +chr21 HAVANA gene 1000 1000000 . + . gene_id "G1"; gene_type "protein_coding"; gene_name "TOUCHES_START"; +chr21 HAVANA gene 1199999 1250000 . + . gene_id "G2"; gene_type "protein_coding"; gene_name "INSIDE"; +chr21 HAVANA gene 1100000 1300000 . + . gene_id "G3"; gene_type "lncRNA"; gene_name "NOT_CODING"; +chr21 HAVANA transcript 1100000 1300000 . + . gene_id "G2"; gene_type "protein_coding"; gene_name "INSIDE"; +chr21 HAVANA gene 1600000 1700000 . + . gene_id "G4"; gene_type "protein_coding"; gene_name "AT_END"; +chr21 HAVANA gene 30500000 30600000 . - . gene_id "G5"; gene_type "protein_coding"; gene_name "DENI"; +chr21 HAVANA gene 1300000 1350000 . - . gene_id "G6"; gene_type "protein_coding"; gene_name "INSIDE"; +chr22 HAVANA gene 17700000 17900000 . + . gene_id "G7"; gene_type "protein_coding"; gene_name "ON22"; +chr22 HAVANA gene 17000000 17100000 . + . gene_id "G8"; gene_type "protein_coding"; gene_id_only "x"; +EOF diff --git a/tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent b/tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-strict/unsorted/exit b/tests/parity/golden/archaic-strict/unsorted/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/archaic-strict/unsorted/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/archaic-strict/unsorted/s.tsv.absent b/tests/parity/golden/archaic-strict/unsorted/s.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-strict/unsorted/stderr b/tests/parity/golden/archaic-strict/unsorted/stderr new file mode 100644 index 000000000..075caa55f --- /dev/null +++ b/tests/parity/golden/archaic-strict/unsorted/stderr @@ -0,0 +1 @@ +ERROR: rows-unsorted.tsv: line 6: position 5001 comes after 5035; the lifted rows must be sorted by position diff --git a/tests/parity/golden/archaic-table/unsorted/t.vcf.absent b/tests/parity/golden/archaic-table/unsorted/t.vcf.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 1604b265c..481cec78f 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -62,3 +62,12 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +archaic-table fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" table "$@"' archaic table +archaic-strict fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" strict "$@"' archaic strict +archaic-genotypes fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" genotypes "$@"' archaic genotypes +archaic-report fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" report "$@"' archaic report +ancient-select fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" select "$@"' ancient select +ancient-lift fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" lift "$@"' ancient lift +ancient-person-geno fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" person-geno "$@"' ancient person-geno +ancient-align fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" align "$@"' ancient align +ancient-report fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" report "$@"' ancient report From cae6e3e598e1a09541f4f4a0ece847490853f564 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:21 +0100 Subject: [PATCH 03/86] refactor(ancestry): delete archaic_report.py and ancient_report.py; tests, demo and evidence run og The step 43 and 44 tests drive og archaic and og ancient as the pipeline does and join Guard's OG_REQUIRED list; their unit checks of internals moved to the Rust tests. The docs demo writes step 44 with og ancient report, archaic_compare.py keeps its own copy of the IBDmix reader, and the archaic-share evidence job builds og. The table and VCF readers no longer build a path string per line. Bead: pgp-bs9u.21 --- .github/workflows/guard.yml | 2 +- .github/workflows/public-evidence.yml | 12 +- bin/ancient_report.py | 753 ------------------------- bin/archaic_compare.py | 26 +- bin/archaic_report.py | 745 ------------------------ bin/collect_summary.py | 2 +- rust/crates/og-ancestry/src/ancient.rs | 3 + rust/crates/og-ancestry/src/archaic.rs | 19 +- scripts/ci/e2e-case-paths.tsv | 4 +- tests/demo/make_demo_sample.py | 81 +-- tests/e2e/ancient-similarity-1.sh | 2 +- tests/test_ancient_report.py | 228 ++++---- tests/test_archaic_report.py | 62 +- 13 files changed, 237 insertions(+), 1702 deletions(-) delete mode 100755 bin/ancient_report.py delete mode 100755 bin/archaic_report.py diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 5a209a286..e22eba988 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -327,7 +327,7 @@ jobs: run: | fail=0 for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py tests/test_run_delta.py \ - tests/test_ancestry_fine_report.py \ + tests/test_ancestry_fine_report.py tests/test_archaic_report.py tests/test_ancient_report.py \ tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ tests/test_str_catalog.py tests/test_str_ranges.py tests/test_og_setup_zip.py tests/test_setup_fine_maps.sh \ tests/test_render_report.py tests/test_collect_summary.py tests/test_chip_report.py \ diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 11b39d5a4..d44d44de8 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -227,7 +227,7 @@ jobs: # The positions of this part where HG002 has two records or more, one # of them an SNV, as GIAB writes the VCF and split with bcftools # norm -m-, and how many of them are rows of the table (the rows the - # same-position rule of archaic_report.py decides). + # same-position rule of og archaic genotypes decides). docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY' @@ -470,7 +470,7 @@ jobs: archaic-share: needs: [archaic-sites, archaic-hg002] runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -492,6 +492,12 @@ jobs: mv parts "lone-${c}" && mkdir parts && mv "lone-${c}" "parts/archaic-part-${c}" fi ls parts + - name: Build og (og archaic report) + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og - name: One report per population, over the autosomes that ran env: WANT: ${{ needs.archaic-sites.outputs.count }} @@ -517,7 +523,7 @@ jobs: mkdir -p "${d}/ibdmix" cp parts/archaic-part-*/"${pop}"/chr*.txt "${d}/ibdmix/" { printf 'chrom\tfirst\tlast\tsites\n'; for s in parts/archaic-part-*/"${pop}"/spans.tsv; do tail -n +2 "$s" | cut -f1-4; done; } > "${d}/spans.tsv" - python3 bin/archaic_report.py report --sample HG002 --ibdmix-dir "${d}/ibdmix" --spans "${d}/spans.tsv" \ + rust/target/release/og archaic report --sample HG002 --ibdmix-dir "${d}/ibdmix" --spans "${d}/spans.tsv" \ --pop "$pop" --out-dir "${d}/out" kv() { awk -F'\t' -v k="$1" '$1 == k {print $2}' "${d}/out/HG002_archaic.tsv"; } [ "$(kv analysed_chromosomes)" -eq "$WANT" ] || { echo "${pop}: $(kv analysed_chromosomes) chromosomes, want ${WANT}"; exit 1; } diff --git a/bin/ancient_report.py b/bin/ancient_report.py deleted file mode 100755 index 82f0002f0..000000000 --- a/bin/ancient_report.py +++ /dev/null @@ -1,753 +0,0 @@ -#!/usr/bin/env python3 -"""Step 44: where a genome falls among ancient genomes on a PCA of present-day -West Eurasians (smartpca, EIGENSOFT), with the ancient samples of the Allen -Ancient DNA Resource (AADR) projected onto the same axes. - -Subcommands, in the order they run: - - select (setup.sh) read the AADR .anno and .ind files and choose the - people: the present-day West Eurasians of MODERN_GROUPS build - the axes, and the ancient West Eurasians with enough sites - covered are projected. Writes the relabelled .ind (everyone - else is Ignore), the population list and ancients.tsv. - lift (setup.sh) move the 1240K sites of the AADR .snp file from - GRCh37 to GRCh38 with UCSC's chain file, and keep a site only - when one of its two alleles is the GRCh38 reference base. - person-geno (step 44) the person's genotype at each site, from bcftools' - output on the gVCF, appended as one more column to the - present-day panel's EIGENSTRAT genotype file. - align (setup.sh, step 44) check that the axes of two smartpca runs - on the same present-day panel agree, up to the sign of each. - report (step 44) the nearest ancient populations, the summary and - the plot. - -Only the Python standard library is used. Every number the report shows comes -from smartpca's .evec files and the AADR annotation. -""" -import argparse -import bisect -import collections -import csv -import gzip -import html -import math -import os -import statistics -import sys - -# Present-day groups of the AADR 1240K file (Group ID, as written in the -# .anno file of v66.p1) that build the axes: Europe, the Caucasus and the Near -# East, from the 1000 Genomes Project, HGDP and SGDP shotgun genomes. North -# African groups (Mozabite) and groups with recent East Eurasian ancestry -# (Saami, Bashkir, Tatar, Yakut) are left out, so the axes describe West -# Eurasian variation only. -MODERN_GROUPS = ( - "Abkhasian", "Adygei", "Albanian", "Armenian", "Basque", "BedouinA", "BedouinB", - "Bulgarian", "CEU", "Chechen", "Czech", "Druze", "English", "Estonian", "FIN", - "Finnish", "French", "GBR", "Georgian", "Greek", "Greek_Crete", "Hungarian", "IBS", - "Icelandic", "Iranian", "Italian_North", "Jew_Iraqi", "Jordanian", "Lezgin", - "Norwegian", "Orcadian", "Ossetian", "Palestinian", "Polish", "Russian", "Sardinian", - "Scottish", "Spanish", "Spanish_North", "TSI", "Turkish", -) -MODERN_DATA_TYPE = "Shotgun.diploid" -# Ancient samples projected: dated, passing the AADR's own assessment, with -# this many of the 1240K autosomal sites covered, inside this box. -MIN_ANCIENT_SNPS = 100_000 -WEST_EURASIA = {"lat": (25.0, 75.0), "lon": (-25.0, 75.0)} -PASSING = ("PASS", "PROVISIONAL_PASS", "MERGE_PASS") -ANCIENT_LABEL = "Ancient" -PERSON_LABEL = "Person" -# PCs the distance uses, each weighted by the square root of its eigenvalue. -DISTANCE_PCS = 4 -TOP = 20 -# Two runs on the same panel give the same axes; any PC whose present-day -# coordinates correlate less than this between the two is a panel mismatch. -MIN_AXIS_CORRELATION = 0.999 -CAVEAT = ("Closeness on a PCA is genetic similarity, not descent: a genome near an ancient group " - "shares ancestry with it, it does not descend from those people.") -QPADM_NOTE = "qpAdm models of ancestry proportions are not part of this step." - -# Header cells of the .anno file this script reads, matched by their start -# (the AADR writes long explanations after the name). -ANNO_COLUMNS = { - "id": "Genetic ID", - "individual": "Individual ID", - "date_bp": "Date mean in BP", - "date_sd": "Date standard deviation in BP", - "full_date": "Full Date", - "group": "Group ID", - "locality": "Locality", - "country": "Political Entity", - "lat": "Latitude", - "lon": "Longitude", - "data_type": "Data type", - "snps": "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", - "assessment": "ASSESSMENT", -} -COMPLEMENT = {"A": "T", "C": "G", "G": "C", "T": "A"} - - -class Fail(Exception): - pass - - -def opener(path): - return gzip.open(path, "rt", errors="replace") if path.endswith(".gz") else open(path, errors="replace") - - -# --- select --------------------------------------------------------------------- - -def anno_columns(header): - cells = [c.strip().strip('"').strip() for c in header] - out = {} - for key, name in ANNO_COLUMNS.items(): - # ASSESSMENT must not match ASSESSMENT WARNINGS: the exact cell first. - hits = [i for i, c in enumerate(cells) if c == name] or [i for i, c in enumerate(cells) if c.startswith(name)] - if key == "assessment": - hits = [i for i in hits if cells[i] == name] - if not hits: - raise Fail(f"the .anno file has no column '{name}'") - out[key] = hits[0] - return out - - -def to_float(s): - try: - v = float(s) - except (TypeError, ValueError): - return None - return v if math.isfinite(v) else None - - -def passing(assessment): - return assessment.strip().upper().startswith(PASSING) - - -def read_anno(path): - with opener(path) as f: - rows = list(csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - if not rows: - raise Fail(f"{path} is empty") - col = anno_columns(rows[0]) - out = [] - for r in rows[1:]: - if len(r) <= max(col.values()): - continue - out.append({k: r[i].strip() for k, i in col.items()}) - return out - - -def choose(anno, min_snps=MIN_ANCIENT_SNPS): - """Return {genetic id: ('modern', group) | ('ancient', row)}: the - present-day reference people and the ancient people to project, one - genetic id per individual (the one with the most sites).""" - modern_groups = set(MODERN_GROUPS) - best = {} - for r in anno: - if not passing(r["assessment"]): - continue - bp = to_float(r["date_bp"]) - snps = to_float(r["snps"]) or 0 - if bp == 0 and r["group"] in modern_groups and r["data_type"] == MODERN_DATA_TYPE: - kind = "modern" - elif bp is not None and bp > 0 and snps >= min_snps: - lat, lon = to_float(r["lat"]), to_float(r["lon"]) - if lat is None or lon is None: - continue - if not (WEST_EURASIA["lat"][0] <= lat <= WEST_EURASIA["lat"][1] - and WEST_EURASIA["lon"][0] <= lon <= WEST_EURASIA["lon"][1]): - continue - kind = "ancient" - else: - continue - key = (kind, r["individual"] or r["id"]) - if key not in best or snps > best[key][0]: - best[key] = (snps, r) - out = {} - for (kind, _), (_, r) in best.items(): - out[r["id"]] = (kind, r["group"] if kind == "modern" else r) - return out - - -def cmd_select(a): - anno = read_anno(a.anno) - chosen = choose(anno, a.min_snps) - n_mod = n_anc = 0 - groups = set() - with open(a.ind) as f, open(a.out_ind + ".tmp", "w") as o: - for line in f: - c = line.split() - if len(c) < 3: - continue - kind, val = chosen.get(c[0], (None, None)) - if kind == "modern": - label = val - n_mod += 1 - groups.add(val) - elif kind == "ancient": - label = ANCIENT_LABEL - n_anc += 1 - else: - label = "Ignore" - o.write(f"{c[0]}\t{c[1]}\t{label}\n") - missing = sorted(set(MODERN_GROUPS) - groups) - if n_mod < 50 or n_anc < 1: - os.remove(a.out_ind + ".tmp") - raise Fail(f"the .ind file matches {n_mod} present-day reference people and {n_anc} ancient ones; " - "is it the .ind file of the same release as the .anno file?") - os.replace(a.out_ind + ".tmp", a.out_ind) - with open(a.poplist, "w") as o: - for g in sorted(groups): - o.write(g + "\n") - with open(a.ancients, "w") as o: - o.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for gid, (kind, r) in sorted(chosen.items()): - if kind != "ancient": - continue - o.write("\t".join(clean(r[k]) for k in ("id", "group", "date_bp", "date_sd", "full_date", - "locality", "country", "lat", "lon", "snps")) + "\n") - print(f"Present-day reference people: {n_mod} in {len(groups)} groups" - + (f" (not in this file: {', '.join(missing)})" if 0 < len(missing) <= 10 else - f" ({len(missing)} of the {len(MODERN_GROUPS)} groups are not in this file)" if missing else "")) - print(f"Ancient people projected: {n_anc} (at least {a.min_snps:,} sites, West Eurasia)") - return 0 - - -def clean(s): - return " ".join(str(s).replace("\t", " ").split()) or "." - - -# --- lift ----------------------------------------------------------------------- - -def read_chain(path): - """UCSC chain file -> {source chrom: sorted [(start, end, chain no, target - chrom, target start, target strand, target size)]}, 0-based half-open - ungapped blocks.""" - blocks = collections.defaultdict(list) - n = -1 - with opener(path) as f: - cur = None - for line in f: - c = line.split() - if not c: - cur = None - continue - if c[0] == "chain": - n += 1 - t_name, t_strand, t_start = c[2], c[4], int(c[5]) - q_name, q_size, q_strand, q_start = c[7], int(c[8]), c[9], int(c[10]) - if t_strand != "+": - raise Fail(f"{path}: chain {c[12] if len(c) > 12 else n} has source strand {t_strand}") - cur = [t_name, t_start, q_name, q_start, q_strand, q_size, n] - continue - if cur is None: - raise Fail(f"{path}: a block line outside a chain: {line.strip()}") - size = int(c[0]) - t_name, t_pos, q_name, q_pos, q_strand, q_size, k = cur - blocks[t_name].append((t_pos, t_pos + size, k, q_name, q_pos, q_strand, q_size)) - if len(c) >= 3: - cur[1] = t_pos + size + int(c[1]) - cur[3] = q_pos + size + int(c[2]) - else: - cur = None - for v in blocks.values(): - v.sort() - return blocks - - -def index_blocks(blocks): - """read_chain's blocks -> {chrom: (blocks, their starts, longest block)}.""" - return {k: (v, [x[0] for x in v], max(x[1] - x[0] for x in v)) for k, v in blocks.items()} - - -def lift_point(index, chrom, pos1): - """1-based GRCh37 position -> (chrom, 1-based position, strand) on the - target, or None when no block, or blocks of more than one chain, cover it.""" - if chrom not in index: - return None - b, starts, longest = index[chrom] - p = pos1 - 1 - hits = [] - # Blocks of different chains can overlap: every block that starts at or - # before p, back to the longest block's length, may still reach it. - j = bisect.bisect_right(starts, p) - 1 - while j >= 0 and b[j][0] > p - longest: - if b[j][1] > p: - hits.append(b[j]) - j -= 1 - if len({h[2] for h in hits}) != 1: - return None - s, _, _, q_name, q_pos, q_strand, q_size = hits[0] - q = q_pos + (p - s) - if q_strand == "-": - q = q_size - 1 - q - return q_name, q + 1, q_strand - - -class Fasta: - """Random access to a FASTA with a samtools .fai index.""" - - def __init__(self, path): - self.f = open(path, "rb") - self.idx = {} - with open(path + ".fai") as fai: - for line in fai: - c = line.split("\t") - self.idx[c[0]] = (int(c[1]), int(c[2]), int(c[3]), int(c[4])) - - def base(self, chrom, pos1): - if chrom not in self.idx: - return None - length, offset, line_bases, line_width = self.idx[chrom] - if not 1 <= pos1 <= length: - return None - p = pos1 - 1 - self.f.seek(offset + (p // line_bases) * line_width + p % line_bases) - return self.f.read(1).decode("ascii", "replace").upper() - - -def eigen_chrom(c): - """The chromosome column of an EIGENSOFT .snp line -> 1..22, or None.""" - try: - n = int(c) - except ValueError: - return None - return n if 1 <= n <= 22 else None - - -def cmd_lift(a): - blocks = index_blocks(read_chain(a.chain)) - fasta = Fasta(a.fasta) - kept = [] - dropped = collections.Counter() - n = 0 - with opener(a.snp) as f: - for line in f: - c = line.split() - if len(c) < 6: - continue - n += 1 - chrom = eigen_chrom(c[1]) - a1, a2 = c[4].upper(), c[5].upper() - if chrom is None: - dropped["not an autosome"] += 1 - continue - if a1 not in COMPLEMENT or a2 not in COMPLEMENT or a1 == a2: - dropped["alleles not two different bases"] += 1 - continue - hit = lift_point(blocks, f"chr{chrom}", int(float(c[3]))) - if hit is None: - dropped["no single place on GRCh38"] += 1 - continue - q_chrom, q_pos, strand = hit - if q_chrom != f"chr{chrom}": - dropped["moves to another chromosome"] += 1 - continue - if strand == "-": - a1, a2 = COMPLEMENT[a1], COMPLEMENT[a2] - ref = fasta.base(q_chrom, q_pos) - if ref not in (a1, a2): - dropped["neither allele is the GRCh38 base"] += 1 - continue - kept.append((chrom, q_pos, c[0], a1, a2)) - kept.sort() - with open(a.out + ".tmp", "w") as o: - o.write("snp\tchrom\tpos\tallele1\tallele2\n") - for chrom, pos, sid, a1, a2 in kept: - o.write(f"{sid}\tchr{chrom}\t{pos}\t{a1}\t{a2}\n") - os.replace(a.out + ".tmp", a.out) - print(f"Sites of the .snp file: {n:,}; on GRCh38: {len(kept):,}") - for why, k in sorted(dropped.items()): - print(f" dropped, {why}: {k:,}") - if not kept: - raise Fail("no site could be placed on GRCh38: check the chain file and the reference") - return 0 - - -# --- person-geno ------------------------------------------------------------------ - -def read_sites(path): - """sites_grch38.tsv -> {(chrom, pos): [(snp, allele1, allele2)]}.""" - out = collections.defaultdict(list) - with open(path) as f: - next(f, None) - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 5: - out[(c[1], int(c[2]))].append((c[0], c[3], c[4])) - return out - - -def dosage(gt, a1, a2): - """Copies of allele1 in a bcftools %TGT genotype, or None (missing, a - haploid or half call, or an allele that is neither of the two).""" - alleles = gt.replace("|", "/").split("/") - if len(alleles) != 2 or any(x not in (a1, a2) for x in alleles): - return None - return sum(1 for x in alleles if x == a1) - - -def person_dosages(sites, genotype_lines): - """bcftools query lines (CHROM POS REF ALT GT) -> ({snp: dosage}, counts).""" - out = {} - counts = collections.Counter() - for line in genotype_lines: - c = line.rstrip("\n").split("\t") - if len(c) < 5: - continue - here = sites.get((c[0], int(c[1]))) - if not here: - continue - for sid, a1, a2 in here: - d = dosage(c[4].upper(), a1, a2) - if d is None: - counts["other_allele"] += 1 - else: - out[sid] = d - counts["called"] += 1 - return out, counts - - -def cmd_person_geno(a): - sites = read_sites(a.sites) - with open(a.genotypes) as f: - dos, counts = person_dosages(sites, f) - n = used = 0 - with open(a.snp) as snp, open(a.moderns_geno) as geno, open(a.out + ".tmp", "w") as o: - width = None - for sline in snp: - c = sline.split() - if not c: - continue - g = geno.readline().rstrip("\n") - if not g: - raise Fail(f"{a.moderns_geno} has fewer lines than {a.snp}") - if width is None: - width = len(g) - elif len(g) != width: - raise Fail(f"{a.moderns_geno} line {n + 1} has {len(g)} genotypes, not {width}") - d = dos.get(c[0]) - if d is not None: - used += 1 - o.write(g + ("9" if d is None else str(d)) + "\n") - n += 1 - if geno.readline(): - raise Fail(f"{a.moderns_geno} has more lines than {a.snp}") - os.replace(a.out + ".tmp", a.out) - with open(a.stats, "w") as o: - o.write(f"panel_sites\t{n}\nsites_genotyped\t{counts['called'] + counts['other_allele']}\n" - f"other_allele\t{counts['other_allele']}\nsites_used\t{used}\n") - print(f"Sites of the panel: {n:,}; genotyped from the gVCF: {counts['called'] + counts['other_allele']:,}; " - f"used: {used:,} (an allele that is neither of a site's two: {counts['other_allele']:,})") - if used == 0: - raise Fail("none of the panel's sites was genotyped: is the gVCF on GRCh38 with chr names?") - return 0 - - -# --- evec, align ------------------------------------------------------------------ - -def read_evec(path): - """smartpca .evec -> (eigenvalues, {id: (label, [pc...])}).""" - evals, rows = [], {} - with open(path) as f: - for line in f: - c = line.split() - if not c: - continue - if c[0].startswith("#"): - if c[0] == "#eigvals:": - evals = [float(x) for x in c[1:]] - continue - rows[c[0]] = (c[-1], [float(x) for x in c[1:-1]]) - if not evals or not rows: - raise Fail(f"{path} is not a smartpca .evec file (no #eigvals: line or no rows)") - return evals, rows - - -def correlation(x, y): - mx, my = statistics.fmean(x), statistics.fmean(y) - sxy = sum((a - mx) * (b - my) for a, b in zip(x, y)) - sx = math.sqrt(sum((a - mx) ** 2 for a in x)) - sy = math.sqrt(sum((b - my) ** 2 for b in y)) - return sxy / (sx * sy) if sx > 0 and sy > 0 else 0.0 - - -def axis_signs(panel_rows, run_rows, n_pcs): - """The sign that turns each of the run's PCs into the panel's, from the - present-day people both runs used to build the axes. A correlation below - MIN_AXIS_CORRELATION on any PC stops: the two are not the same axes.""" - shared = sorted(k for k, (lab, _) in run_rows.items() - if k in panel_rows and lab not in (ANCIENT_LABEL, PERSON_LABEL) - and panel_rows[k][0] == lab) - if len(shared) < 10: - raise Fail(f"only {len(shared)} present-day people are in both runs; the panel and this run do not match") - signs = [] - for k in range(n_pcs): - r = correlation([panel_rows[i][1][k] for i in shared], [run_rows[i][1][k] for i in shared]) - if abs(r) < MIN_AXIS_CORRELATION: - raise Fail(f"PC{k + 1} of this run correlates {r:.4f} with the panel's over {len(shared)} present-day " - "people: the axes differ. Install the panel again (setup.sh --ancient-panel).") - signs.append(1.0 if r > 0 else -1.0) - return signs - - -def cmd_align(a): - _, panel = read_evec(a.panel_evec) - _, run = read_evec(a.run_evec) - signs = axis_signs(panel, run, a.pcs) - print("Axes agree with the panel's: " + " ".join(f"PC{i + 1}{'+' if s > 0 else '-'}" for i, s in enumerate(signs))) - return 0 - - -# --- report ----------------------------------------------------------------------- - -def year_label(bp): - """A date in years before 1950 -> 'N BCE' or 'N CE', the AADR's own - reading (its 2600-2000 BCE has a mean of 4250 BP): BCE = BP - 1950 and - CE = 1950 - BP, with 1950 BP read as 1 BCE.""" - y = 1950 - int(round(bp)) - return f"{y} CE" if y >= 1 else f"{max(-y, 1)} BCE" - - -def range_label(old_bp, young_bp): - """The span of a population's dates, oldest first, e.g. '5300-4900 BCE', - '200-450 CE', or '300 BCE-200 CE'.""" - a, b = year_label(old_bp), year_label(young_bp) - if a == b: - return a - (na, ea), (nb, eb) = a.split(" "), b.split(" ") - return f"{na}-{nb} {ea}" if ea == eb else f"{a}-{b}" - - -def read_ancients(path): - out = {} - with open(path) as f: - for r in csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE): - out[r["id"]] = r - return out - - -def weights(evals, n): - return [math.sqrt(max(v, 0.0)) for v in evals[:n]] - - -def distance(p, q, w): - return math.sqrt(sum((wi * (a - b)) ** 2 for a, b, wi in zip(p, q, w))) - - -def most_common(values): - vals = [v for v in values if v and v != "."] - if not vals: - return "." - c = collections.Counter(vals).most_common() - top = [v for v, k in c if k == c[0][1]] - return sorted(top)[0] - - -def nearest_populations(person, panel_rows, ancients, evals, n_pcs=DISTANCE_PCS, top=TOP): - """Rank the ancient groups by the weighted distance from the person to the - group's mean position on the first n_pcs PCs.""" - w = weights(evals, n_pcs) - groups = collections.defaultdict(list) - for gid, r in ancients.items(): - if gid in panel_rows and panel_rows[gid][0] == ANCIENT_LABEL: - groups[r["group"]].append((panel_rows[gid][1], r)) - rows = [] - for g, members in groups.items(): - k = len(members[0][0]) - centre = [statistics.fmean(m[0][i] for m in members) for i in range(min(k, n_pcs))] - dates = [to_float(m[1]["date_bp"]) for m in members] - dates = [d for d in dates if d is not None] - rows.append({ - "group": g, - "n": len(members), - "distance": distance(person[:n_pcs], centre, w), - "pc1": centre[0], - "pc2": centre[1] if len(centre) > 1 else 0.0, - "date_bp": statistics.median(dates) if dates else None, - "dates": range_label(max(dates), min(dates)) if dates else ".", - "country": most_common(m[1]["country"] for m in members), - "locality": most_common(m[1]["locality"] for m in members), - }) - rows.sort(key=lambda r: (r["distance"], r["group"])) - for i, r in enumerate(rows, 1): - r["rank"] = i - return rows[:top], len(rows) - - -AGE_BINS = ( # (oldest BP, colour, legend), oldest first; dark-background palette - (math.inf, "#5aa9e6", "before 8000 BCE"), - (9950, "#7fc97f", "8000-4000 BCE"), - (5950, "#f0a35e", "4000-1000 BCE"), - (2950, "#e377c2", "after 1000 BCE"), -) - - -def age_colour(bp): - colour = AGE_BINS[0][1] - for limit, c, _ in AGE_BINS: - if bp is not None and bp < limit: - colour = c - return colour - - -def svg_plot(person, panel_rows, ancients, sample, nearest, width=900, height=720): - """The PC1/PC2 plane: present-day people grey, ancient people coloured by - age, the person a large ring. Dark background, so it reads in a dark - report and on its own.""" - pts = [v[1][:2] for v in panel_rows.values()] + [person[:2]] - xs, ys = [p[0] for p in pts], [p[1] for p in pts] - pad = 0.04 - x0, x1 = min(xs), max(xs) - y0, y1 = min(ys), max(ys) - dx, dy = (x1 - x0) or 1.0, (y1 - y0) or 1.0 - left, right, top, bottom = 60, 20, 50, 110 - pw, ph = width - left - right, height - top - bottom - - def X(v): - return left + ((v - x0) / dx * (1 - 2 * pad) + pad) * pw - - def Y(v): - return top + (1 - ((v - y0) / dy * (1 - 2 * pad) + pad)) * ph - - e = html.escape - out = [f'', - '', - f'', - f'{e(sample)} among ancient West Eurasians ' - '(AADR, projected onto present-day PCs)', - f''] - for label, pcs in panel_rows.values(): - if label != ANCIENT_LABEL: - out.append(f'') - for gid, (label, pcs) in panel_rows.items(): - if label == ANCIENT_LABEL and gid in ancients: - c = age_colour(to_float(ancients[gid]["date_bp"])) - out.append(f'') - for r in nearest[:5]: - out.append(f'{e(r["group"])}') - px, py = X(person[0]), Y(person[1]) - out.append(f'') - out.append(f'') - out.append(f'{e(sample)}') - out.append(f'PC1') - out.append(f'PC2') - lx, ly = left, top + ph + 48 - out.append(f'present day') - lx += 120 - for _, c, name in AGE_BINS: - out.append(f'{e(name)}') - lx += 150 - out.append(f'{e(CAVEAT)}') - out.append("") - return "\n".join(out) + "\n" - - -def cmd_report(a): - evals, panel = read_evec(a.panel_evec) - _, run = read_evec(a.run_evec) - if a.sample not in run or run[a.sample][0] != PERSON_LABEL: - raise Fail(f"{a.run_evec} has no row for {a.sample} labelled {PERSON_LABEL}") - n_pcs = min(DISTANCE_PCS, len(evals), len(run[a.sample][1])) - signs = axis_signs(panel, run, max(2, n_pcs)) - person = [s * v for s, v in zip(signs, run[a.sample][1])] - ancients = read_ancients(a.ancients) - nearest, n_groups = nearest_populations(person, panel, ancients, evals, n_pcs) - stats = {} - if a.stats: - with open(a.stats) as f: - for line in f: - k, _, v = line.rstrip("\n").partition("\t") - stats[k] = v - n_anc = sum(1 for lab, _ in panel.values() if lab == ANCIENT_LABEL) - n_mod = sum(1 for lab, _ in panel.values() if lab not in (ANCIENT_LABEL, PERSON_LABEL)) - with open(a.out_tsv + ".tmp", "w") as o: - o.write("rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n") - for r in nearest: - o.write(f"{r['rank']}\t{clean(r['group'])}\t{r['n']}\t{r['distance']:.5f}\t{r['pc1']:.5f}\t{r['pc2']:.5f}\t" - f"{'.' if r['date_bp'] is None else int(round(r['date_bp']))}\t{r['dates']}\t" - f"{clean(r['country'])}\t{clean(r['locality'])}\n") - os.replace(a.out_tsv + ".tmp", a.out_tsv) - info = [ - ("release", a.release), - ("sites_used", stats.get("sites_used", ".")), - ("panel_sites", stats.get("panel_sites", ".")), - ("present_day_people", n_mod), - ("ancient_people", n_anc), - ("ancient_groups", n_groups), - ("pcs_in_distance", n_pcs), - ("pc1", f"{person[0]:.5f}"), - ("pc2", f"{person[1]:.5f}"), - ] - with open(a.out_info + ".tmp", "w") as o: - o.write("key\tvalue\n") - for k, v in info: - o.write(f"{k}\t{v}\n") - os.replace(a.out_info + ".tmp", a.out_info) - with open(a.out_svg + ".tmp", "w") as o: - o.write(svg_plot(person, panel, ancients, a.sample, nearest)) - os.replace(a.out_svg + ".tmp", a.out_svg) - lines = [f"Ancient populations nearest to {a.sample} (AADR {a.release}, smartpca with lsqproject)", - f"Sites used: {stats.get('sites_used', '.')} of {stats.get('panel_sites', '.')}; axes from {n_mod} " - f"present-day West Eurasians; {n_anc} ancient people in {n_groups} groups projected.", - f"Distance: the first {n_pcs} PCs, each weighted by the square root of its eigenvalue.", ""] - for r in nearest[:10]: - lines.append(f" {r['rank']:>2}. {r['group']:<45} {r['dates']:<22} {r['country']:<20} n={r['n']:<4} " - f"d={r['distance']:.4f}") - lines += ["", CAVEAT, QPADM_NOTE] - with open(a.out_txt, "w") as o: - o.write("\n".join(lines) + "\n") - print("\n".join(lines)) - return 0 - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - sub = p.add_subparsers(dest="cmd", required=True) - s = sub.add_parser("select") - s.add_argument("--anno", required=True) - s.add_argument("--ind", required=True) - s.add_argument("--out-ind", required=True) - s.add_argument("--poplist", required=True) - s.add_argument("--ancients", required=True) - s.add_argument("--min-snps", type=int, default=MIN_ANCIENT_SNPS) - s = sub.add_parser("lift") - s.add_argument("--snp", required=True) - s.add_argument("--chain", required=True) - s.add_argument("--fasta", required=True) - s.add_argument("--out", required=True) - s = sub.add_parser("person-geno") - s.add_argument("--sites", required=True) - s.add_argument("--genotypes", required=True) - s.add_argument("--snp", required=True) - s.add_argument("--moderns-geno", required=True) - s.add_argument("--out", required=True) - s.add_argument("--stats", required=True) - s = sub.add_parser("align") - s.add_argument("--panel-evec", required=True) - s.add_argument("--run-evec", required=True) - s.add_argument("--pcs", type=int, default=DISTANCE_PCS) - s = sub.add_parser("report") - s.add_argument("--panel-evec", required=True) - s.add_argument("--run-evec", required=True) - s.add_argument("--ancients", required=True) - s.add_argument("--sample", required=True) - s.add_argument("--release", required=True) - s.add_argument("--stats") - s.add_argument("--out-tsv", required=True) - s.add_argument("--out-info", required=True) - s.add_argument("--out-txt", required=True) - s.add_argument("--out-svg", required=True) - a = p.parse_args(argv) - try: - return {"select": cmd_select, "lift": cmd_lift, "person-geno": cmd_person_geno, - "align": cmd_align, "report": cmd_report}[a.cmd](a) - except Fail as e: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/archaic_compare.py b/bin/archaic_compare.py index 88cde4e31..7c6ebdd7d 100644 --- a/bin/archaic_compare.py +++ b/bin/archaic_compare.py @@ -24,8 +24,25 @@ import statistics import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import archaic_report # noqa: E402 (the same reader and cutoffs as step 43) +ARCHAICS = ("Altai", "Vindija", "Denisova") + + +def ibdmix_segments(path, min_lod=4.0, min_length=50000): + """IBDmix's segments for SAMPLE in one of its output files, kept as step 43 + keeps them (og archaic report: LOD 4 and 50 kb or longer), as (start, end).""" + out = [] + with open(path) as f: + head = f.readline().rstrip("\n").split("\t") + if head[:5] != ["ID", "chrom", "start", "end", "slod"]: + sys.exit(f"ERROR: {path} is not IBDmix output (header {' '.join(head[:5])})") + for line in f: + c = line.rstrip("\n").split("\t") + if len(c) < 5 or c[0] != "SAMPLE": + continue + s, e, lod = int(c[2]), int(c[3]), float(c[4]) + if lod >= min_lod and e - s >= min_length: + out.append((s, e)) + return out def norm(c): @@ -170,9 +187,8 @@ def cmd_genomes(a): chroms = list(dict.fromkeys(norm(line.split("\t")[0]) for line in f if line.strip())) per = collections.defaultdict(lambda: collections.defaultdict(list)) for c in chroms: - for n in archaic_report.ARCHAICS: - segs = archaic_report.keep(archaic_report.read_segments(os.path.join(a.ibdmix_dir, f"chr{c}.{n}.txt"))) - per[n][c] += [(s, e) for _, s, e, _ in segs] + for n in ARCHAICS: + per[n][c] += ibdmix_segments(os.path.join(a.ibdmix_dir, f"chr{c}.{n}.txt")) per["Altai+Vindija"] = {c: per["Altai"][c] + per["Vindija"][c] for c in chroms} released = sum(merged_bp(calls[a.sample].get(c, [])) for c in chroms) print(f"| IBDmix output | bp | under the released {a.sample} calls | only here | only released |") diff --git a/bin/archaic_report.py b/bin/archaic_report.py deleted file mode 100755 index 6ada76bd2..000000000 --- a/bin/archaic_report.py +++ /dev/null @@ -1,745 +0,0 @@ -#!/usr/bin/env python3 -"""archaic_report.py: Neanderthal and Denisovan segments in one genome, with IBDmix. - -IBDmix (Chen et al. 2020, Cell 180:677, doi:10.1016/j.cell.2020.01.012) compares -a modern genome with an archaic one site by site and adds up a log-odds score -(LOD) that a stretch of the modern genome shares a recent ancestor with the -archaic genome, against the chance of the same genotypes in the modern -population. It needs no unadmixed reference population, but it does need the -allele frequency of each site in the modern population, and it takes that from -the modern people in its input. With one genome as the only modern input the -frequency is that person's own genotype and no site scores (find_frequency in -IBDmix's src/Genotype_Reader.cc drops every site below its minor allele count), -so `genotypes` adds PSEUDO stand-ins whose genotypes carry the 1000 Genomes -allele count of the chosen population at each site. IBDmix scores each modern -column from the site's frequency and that column's genotype only, so your rows -are the ones it would give you inside a population sample of that frequency; -the stand-ins' own rows are dropped. - -Subcommands: - - table (setup.sh --archaic) one GRCh37 chromosome: the 1000 Genomes - phase 3 biallelic SNVs with their allele frequencies, and the - genotypes of the Altai Neanderthal, the Vindija 33.19 Neanderthal - and the Altai Denisovan at those sites and wherever one of the - three carries a non-reference allele, as a sites-only VCF that - Picard lifts to GRCh38. Each archaic genome is read inside its - own mask (--mask NAME=BED, the MPI-EVA FilterBed of that - genome): a call outside it is kept as masked, code 10 plus its - ALT count. Every call at a site inside an excluded region - (--exclude BED: segmental duplications, the 5 bp around a modern - indel) is masked the same way. That is how Chen et al. filter: - the site stays in IBDmix's input and IBDmix's --mask stops it - scoring, except that a homozygous mismatch there still counts - against a segment. - strict (setup.sh --archaic) the lifted GRCh38 rows of one chromosome with - every row outside the 1000 Genomes strict mask marked masked in - all three archaic columns (the same code 10 to 12), instead of - dropped, so IBDmix still scores a mismatch there. - genotypes (step 43) IBDmix's genotype file of each chromosome: the three - archaic columns, SAMPLE (yours, from the VCF) and the stand-ins; - the mask of each archaic genome (chrN..mask.bed, the - positions where its call is masked); and spans.tsv, the span and - site count of each chromosome. - report (step 43) IBDmix's segments for SAMPLE against each archaic genome, - kept at LOD 4 and 50 kb or longer (the cutoffs of Chen et al.), - merged into Neanderthal and Denisovan segments, with the share of - the analysed span, the ten longest segments and the protein-coding - genes in them. - -Codes: an archaic or modern genotype is the number of ALT alleles (0, 1, 2), or -9 when it is unknown, the coding IBDmix's genotype file uses. In the table an -archaic call outside that genome's mask is 10, 11 or 12 (MASKED plus the ALT -count); `genotypes` writes it to IBDmix as 0, 1 or 2 and lists the position in -that genome's mask file (chrN..mask.bed), which step 43 passes to -IBDmix with -r. - -Used by scripts/setup.sh, scripts/43-archaic.sh and the ARCHAIC_* processes -(bin/ is on the task PATH); tests/test_archaic_report.py checks it. Standard -library only. -""" -import argparse -import bisect -import gzip -import heapq -import os -import re -import sys - -ARCHAICS = ("Altai", "Vindija", "Denisova") -NEANDERTHALS = ("Altai", "Vindija") -POPS = ("ALL", "AFR", "AMR", "EAS", "EUR", "SAS") -TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova"] + [f"AF_{p}" for p in POPS] -MASKED = 10 # table code: MASKED + ALT count, a call outside the genome's mask -PSEUDO = 100 # stand-in modern people: 200 alleles, about one 1000 Genomes population -MIN_LOD = 4.0 # Chen et al. 2020: LOD 4 -MIN_LENGTH = 50000 # and 50 kb -TOP = 10 -SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] -CAVEAT = ("IBDmix finds stretches where your genome matches a Neanderthal or the Denisovan genome more " - "closely than the allele frequencies of a modern population explain. A stretch counts when either " - "of your two copies carries it, and the share is of the analysed span of the autosomes. Some of what " - "it finds in anyone is old ancestry shared with the archaic genomes rather than gene flow from them: " - "Chen et al. 2020 find Neanderthal sequence in every population they studied, Africans included. " - "The numbers move with the cutoffs (LOD 4, 50 kb) and with the frequencies used. An estimate, not " - "a measurement.") - - -def die(msg, code=1): - print(f"ERROR: {msg}", file=sys.stderr) - sys.exit(code) - - -def opener(path): - return gzip.open(path, "rt") if path.endswith(".gz") else open(path) - - -def gt_code(gt): - """'0/1', '1|1', '0/0' -> 1, 2, 0; a missing, partial or multi-allelic call -> None.""" - a = re.split(r"[/|]", gt) - if len(a) != 2 or any(x not in ("0", "1") for x in a): - return None - return int(a[0]) + int(a[1]) - - -# --- table (setup) ------------------------------------------------------------------ - -def read_modern(path): - """1000 Genomes lines: POS REF ALT AF AFR_AF AMR_AF EAS_AF EUR_AF SAS_AF (bcftools query).""" - with opener(path) as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 9: - die(f"{path}: a line has {len(c)} columns, want 9: {line[:80]!r}") - if len(c[1]) != 1 or len(c[2]) != 1: - continue - yield int(c[0]), 0, (c[1], c[2], c[3:9]) - - -def read_archaic(path, idx): - """Archaic lines: POS REF ALT GT (bcftools query).""" - with opener(path) as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 4: - die(f"{path}: a line has {len(c)} columns, want 4: {line[:80]!r}") - yield int(c[0]), 1 + idx, (c[1], c[2], c[3]) - - -def read_mask(path, chrom): - """The intervals of one chromosome of a mask BED, sorted and merged. - A BED is 0-based and half-open, so [start, end) holds the 1-based - positions start + 1 to end. The chromosome is matched with and without - the chr prefix, because the archaic masks name it 22 and others chr22.""" - num = str(chrom)[3:] if str(chrom).startswith("chr") else str(chrom) - want = {num, f"chr{num}"} - iv = [] - with opener(path) as f: - for line in f: - if line.startswith(("#", "track", "browser")): - continue - c = line.split() - if len(c) < 3 or c[0] not in want: - continue - s, e = int(c[1]), int(c[2]) - if e <= s: - continue # an empty or reversed interval holds no position - iv.append((s, e)) - iv.sort() - out = [] - for s, e in iv: - if out and s <= out[-1][1]: - out[-1][1] = max(out[-1][1], e) - else: - out.append([s, e]) - return out - - -class Mask: - """Whether a 1-based position is inside a mask. Positions must come in - ascending order, as they do in the table build: the walk over the - intervals never goes back.""" - - def __init__(self, intervals): - self.iv = intervals - self.i = 0 - - def holds(self, pos): - p = pos - 1 - while self.i < len(self.iv) and self.iv[self.i][1] <= p: - self.i += 1 - return self.i < len(self.iv) and self.iv[self.i][0] <= p - - -def archaic_code(rec, ref, alt): - """The ALT count of one archaic record at a row with this REF and ALT, or 9.""" - if rec is None: - return 9 - aref, aalt, gt = rec - if aref != ref: - return 9 - code = gt_code(gt) - if code is None: - return 9 - if aalt in (".", "<*>") or code == 0: - return 0 if code == 0 else 9 - return code if aalt == alt else 9 - - -def table_row(pos, modern, arch): - """The row of one position, or None. modern: (REF, ALT, AFs) or None; - arch: one (REF, ALT, GT) or None per archaic genome.""" - if modern is not None: - ref, alt, afs = modern - else: - # A site no modern person varies at: the ALT of the first archaic - # genome with a non-reference call, every population frequency 0. - cand = [r for r in arch if r is not None and gt_code(r[2]) not in (None, 0) - and len(r[0]) == 1 and len(r[1]) == 1] - if not cand: - return None - ref, alt, afs = cand[0][0], cand[0][1], ["0"] * len(POPS) - codes = [archaic_code(r, ref, alt) for r in arch] - if all(c == 9 for c in codes): - return None - if modern is None and not any(c in (1, 2) for c in codes): - return None - clean = [] - for v in afs: - try: - x = float(v) - except ValueError: - x = 0.0 - clean.append(f"{min(max(x, 0.0), 1.0):.6g}") - return ref, alt, codes, clean - - -def cmd_table(a): - names = [] - streams = [read_modern(a.modern)] - for i, spec in enumerate(a.archaic): - name, _, path = spec.partition("=") - if name not in ARCHAICS or not path: - die(f"--archaic takes NAME=FILE with NAME one of {', '.join(ARCHAICS)}, got {spec!r}", 2) - names.append(name) - streams.append(read_archaic(path, i)) - if names != list(ARCHAICS): - die(f"--archaic must name {', '.join(ARCHAICS)} in that order, got {', '.join(names)}", 2) - masks = {} - for spec in a.mask: - name, _, path = spec.partition("=") - if name not in ARCHAICS or not path: - die(f"--mask takes NAME=FILE with NAME one of {', '.join(ARCHAICS)}, got {spec!r}", 2) - if name in masks: - die(f"--mask names {name} twice", 2) - iv = read_mask(path, a.chrom) - if not iv: - die(f"{path} holds no interval on chromosome {a.chrom}: a mask that matches nothing " - "would make every site unknown") - masks[name] = Mask(iv) - excluded = [] - for path in a.exclude: - iv = read_mask(path, a.chrom) - if not iv: - die(f"{path} holds no interval on chromosome {a.chrom}: an empty exclusion list is a " - "download that went wrong, not a chromosome without segmental duplications") - excluded.append(Mask(iv)) - chrom = a.chrom if a.chrom.startswith("chr") else f"chr{a.chrom}" - out = open(a.out, "w") - out.write("##fileformat=VCFv4.2\n") - out.write(f"##contig=\n") - for n in ARCHAICS: - out.write(f'##INFO=\n') - for p in POPS: - out.write(f'##INFO=\n') - out.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n") - rows = 0 - cur, modern, arch, last = None, None, [None] * len(ARCHAICS), -1 - - def flush(): - nonlocal rows - if cur is None: - return - # Every Mask is walked for every position, before any early return, - # because the walk only goes forward. - excl = any([m.holds(cur) for m in excluded]) - outside = [excl or (n in masks and not masks[n].holds(cur)) for n in ARCHAICS] - r = table_row(cur, modern, arch) - if r is None: - return - ref, alt, codes, afs = r - # A genome's call outside its mask, or at an excluded site, is kept but - # marked masked (MASKED + code). Step 43 hands IBDmix the call and a - # mask of those positions, as Chen et al. do: IBDmix scores nothing - # there except a homozygous mismatch, which still counts against a - # segment. Dropping the call instead (code 9) let segments run through - # masked sequence nothing could contradict. - codes = [c + MASKED if (out and c != 9) else c for c, out in zip(codes, outside)] - info = ";".join([f"{n}={c}" for n, c in zip(ARCHAICS, codes)] + [f"AF_{p}={v}" for p, v in zip(POPS, afs)]) - out.write(f"{chrom}\t{cur}\t.\t{ref}\t{alt}\t.\t.\t{info}\n") - rows += 1 - - for pos, src, rec in heapq.merge(*streams, key=lambda t: (t[0], t[1])): - if pos < last: - die(f"input not sorted by position at {pos}") - if pos != cur: - flush() - cur, modern, arch = pos, None, [None] * len(ARCHAICS) - last = pos - if src == 0: - # A second modern record at one position (a split multi-allelic - # site) makes the row ambiguous: keep neither. - modern = rec if modern is None else ("N", "N", ["0"] * len(POPS)) - else: - arch[src - 1] = rec - flush() - out.close() - print(f"{chrom}: {rows} rows", file=sys.stderr) - - -def cmd_strict(a): - """The lifted GRCh38 rows of one chromosome (no header, the 13 table - columns), once per position, with the calls of every row outside the 1000 - Genomes strict mask marked masked (MASKED plus the ALT count, 9 stays 9). - The strict mask is the same for the three genomes, so all three are - marked; step 43 then hands IBDmix those positions as its mask.""" - iv = read_mask(a.strict, a.chrom) - if not iv: - die(f"{a.strict} holds no interval on chromosome {a.chrom}: a mask that matches nothing " - "would mark every site masked") - starts = [s for s, _ in iv] - seen = set() - rows = marked = 0 - with open(a.rows) as f, open(a.out, "w") as out: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) != len(TABLE_HEADER): - die(f"{a.rows}: a row has {len(c)} columns, want {len(TABLE_HEADER)}: {line[:80]!r}") - pos = int(c[1]) - if pos in seen: - continue - seen.add(pos) - i = bisect.bisect_right(starts, pos - 1) - 1 - if not (i >= 0 and pos - 1 < iv[i][1]): - hit = False - for k in (4, 5, 6): - if c[k] in ("0", "1", "2"): - c[k] = str(int(c[k]) + MASKED) - hit = True - marked += hit - out.write("\t".join(c) + "\n") - rows += 1 - print(f"{a.chrom}: {rows} rows, {marked} outside the strict mask and marked masked", file=sys.stderr) - - -# setup.sh --archaic stamps its table with the sha256 of everything above this -# section, so an edit below it does not rebuild the table (about 165 GB of downloads). - -# --- genotypes (step 43) ---------------------------------------------------------- - -def pseudo_columns(n): - """The stand-in columns for each ALT allele count 0..2n: as many 2s and - one 1 as the count needs, then 0s.""" - cols = [] - for ac in range(2 * n + 1): - g = ["2"] * (ac // 2) + (["1"] if ac % 2 else []) - g += ["0"] * (n - len(g)) - cols.append("\t".join(g)) - return cols - - -def read_vcf(path): - """(chrom, pos, ref, alt, code) of the biallelic SNVs of a one-sample VCF. - A record whose FILTER is not PASS or '.' gives code 9 (not read), so a - DeepVariant RefCall is neither your allele nor the reference.""" - with opener(path) as f: - for line in f: - if line.startswith("##"): - continue - c = line.rstrip("\n").split("\t") - if line.startswith("#"): - if len(c) != 10: - die(f"{path} holds {max(len(c) - 9, 0)} samples; step 43 reads a one-sample VCF " - "(keep one with bcftools view -s ).") - continue - if len(c) < 10 or len(c[3]) != 1 or c[4] == ".": - continue - if "," in c[4] or len(c[4]) != 1: - yield c[0], int(c[1]), c[3], None, 9 - continue - fmt = c[8].split(":") - if "GT" not in fmt: - continue - vals = c[9].split(":") - gt = vals[fmt.index("GT")] if fmt.index("GT") < len(vals) else "." - code = gt_code(gt) - if code is None or c[6] not in ("PASS", "."): - code = 9 - yield c[0], int(c[1]), c[3], c[4], code - - -CLASH = object() -ARCHAIC_CODES = {str(v) for v in (0, 1, 2, 9, MASKED, MASKED + 1, MASKED + 2)} - - -def pick_call(prev, call): - """The call to keep when a position comes twice, as in a VCF split with - bcftools norm -m-: each is (REF, ALT, code), ALT None for an insertion or a - multi-allelic record. An SNV wins over such a record, in either order. Of - one SNV twice, a usable record wins over code 9, and two different usable - codes are CLASH. Of two different SNVs, the - one that carries its ALT wins over a 0/0 or unusable one (a split 0/1 at - A>G,T is A>G 0/1 and A>T 0/0); both 0/0 is the reference; both carrying - their ALT, or neither readable, is CLASH (unknown, 9).""" - if prev is None: - return call - if prev[1] is None or call[1] is None: - return call if prev[1] is None and call[1] is not None else prev - if call[1] == prev[1]: - if call[2] == 9 or call[2] == prev[2]: - return prev - return call if prev[2] == 9 else CLASH - has_prev, has_call = prev[2] in (1, 2), call[2] in (1, 2) - if has_prev != has_call: - return prev if has_prev else call - if not has_prev and prev[2] == 0 and call[2] == 0: - return (prev[0], None, 0) - return CLASH - - -def read_table(path): - """Rows of one chromosome of the GRCh38 table, with the header checked.""" - with opener(path) as f: - head = f.readline().rstrip("\n").split("\t") - if head != TABLE_HEADER: - die(f"{path} does not start with the table header ({' '.join(TABLE_HEADER)})") - for line in f: - c = line.rstrip("\n").split("\t") - yield c - - -def cmd_genotypes(a): - if a.pop not in POPS: - die(f"--pop must be one of {', '.join(POPS)}, got {a.pop!r}", 2) - af_col = TABLE_HEADER.index(f"AF_{a.pop}") - tables = {} - for name in sorted(os.listdir(a.table_dir)): - m = re.fullmatch(r"(chr(?:[1-9]|1[0-9]|2[0-2]))\.tsv\.gz", name) - if m: - tables[m.group(1)] = os.path.join(a.table_dir, name) - if not tables: - die(f"{a.table_dir} holds no chrN.tsv.gz: install the archaic data with scripts/setup.sh --archaic") - os.makedirs(a.out_dir, exist_ok=True) - pseudo = pseudo_columns(a.pseudo) - two_n = 2 * a.pseudo - head = "\t".join(["chrom", "pos", "ref", "alt", *ARCHAICS, "SAMPLE"] + [f"P{i:03d}" for i in range(1, a.pseudo + 1)]) - with open(os.path.join(a.out_dir, "samples.txt"), "w") as f: - f.write("SAMPLE\n" + "".join(f"P{i:03d}\n" for i in range(1, a.pseudo + 1))) - spans = [] - vcf = read_vcf(a.vcf) - pending = next(vcf, None) - done = set() - while pending is not None: - chrom = pending[0] - if chrom in done: - die(f"{a.vcf} is not sorted: {chrom} comes back after another chromosome") - done.add(chrom) - if chrom not in tables: - while pending is not None and pending[0] == chrom: - pending = next(vcf, None) - continue - # The sample's calls on this chromosome, merged with the table by position. - calls, clash = {}, set() - while pending is not None and pending[0] == chrom: - pos = pending[1] - if pos not in clash: - calls[pos] = pick_call(calls.get(pos), pending[2:]) - if calls[pos] is CLASH: - calls[pos] = (pending[2], None, 9) - clash.add(pos) - pending = next(vcf, None) - n = first = last = dropped = carried = 0 - num = chrom[3:] - path = os.path.join(a.out_dir, f"{chrom}.gt.gz") - # Per archaic genome, the runs of written rows whose call is masked: - # [first, last] positions, closed at the next unmasked row. - runs = {g: [] for g in ARCHAICS} - open_run = {g: None for g in ARCHAICS} - with gzip.open(path, "wt", compresslevel=1) as out: - out.write(head + "\n") - for c in read_table(tables[chrom]): - pos, ref, alt = int(c[1]), c[2], c[3] - arch = [] - for g, v in zip(ARCHAICS, c[4:7]): - if v not in ARCHAIC_CODES: - die(f"{tables[chrom]}: {chrom}:{pos} has {g} code {v!r}; the table holds 0, 1, 2, 9 " - f"or {MASKED} to {MASKED + 2}") - arch.append(int(v)) - call = calls.get(pos) - if call is None: - code = 0 - elif call[0] != ref or (call[1] is not None and call[1] != alt): - # Another allele than the row's: IBDmix's generate_gt drops the line. - dropped += 1 - continue - else: - code = call[2] - if code in (1, 2): - carried += 1 - try: - ac = int(round(float(c[af_col]) * two_n)) - except ValueError: - ac = 0 - ac = min(max(ac, 0), two_n) - for g, v in zip(ARCHAICS, arch): - if v >= MASKED: - if open_run[g] is None: - open_run[g] = [pos, pos] - else: - open_run[g][1] = pos - elif open_run[g] is not None: - runs[g].append(open_run[g]) - open_run[g] = None - gts = "\t".join(str(v - MASKED if v >= MASKED else v) for v in arch) - out.write(f"{num}\t{pos}\t{ref}\t{alt}\t{gts}\t{code}\t{pseudo[ac]}\n") - if n == 0: - first = pos - last = pos - n += 1 - if n == 0: - os.remove(path) - continue - # IBDmix's mask (-r): chromosome as in the genotype file, and a line - # covers the positions start + 1 to end, as in a BED. - for g in ARCHAICS: - if open_run[g] is not None: - runs[g].append(open_run[g]) - with open(os.path.join(a.out_dir, f"{chrom}.{g}.mask.bed"), "w") as f: - f.writelines(f"{num}\t{s0 - 1}\t{e0}\n" for s0, e0 in runs[g]) - spans.append((chrom, first, last, n, carried, dropped)) - with open(os.path.join(a.out_dir, "spans.tsv"), "w") as f: - f.write("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n") - for s in spans: - f.write("\t".join(map(str, s)) + "\n") - if not spans: - die(f"{a.vcf} has no records on a chromosome of the archaic table ({', '.join(sorted(tables))})") - print(f"{len(spans)} chromosomes, {sum(s[3] for s in spans)} sites", file=sys.stderr) - - -# --- report (step 43) ------------------------------------------------------------ - -def read_segments(path, sample="SAMPLE"): - """IBDmix's output rows for SAMPLE: (chrom, start, end, lod).""" - segs = [] - with open(path) as f: - head = f.readline().rstrip("\n").split("\t") - if head[:5] != ["ID", "chrom", "start", "end", "slod"]: - die(f"{path} is not IBDmix output (header {' '.join(head[:5])})") - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 5 or c[0] != sample: - continue - segs.append((c[1], int(c[2]), int(c[3]), float(c[4]))) - return segs - - -def keep(segs, min_lod=MIN_LOD, min_length=MIN_LENGTH): - return [s for s in segs if s[3] >= min_lod and s[2] - s[1] >= min_length] - - -def merge(segs): - """Overlapping or touching segments of one class as one: (chrom, start, end, lod, members).""" - out = [] - for chrom, start, end, lod, who in sorted(segs, key=lambda s: (chrom_key(s[0]), s[1], s[2])): - if out and out[-1][0] == chrom and start <= out[-1][2]: - c, s0, e0, l0, w0 = out[-1] - out[-1] = (c, s0, max(e0, end), max(l0, lod), w0 | who) - else: - out.append((chrom, start, end, lod, set(who))) - return out - - -def chrom_key(c): - c = str(c) - c = c[3:] if c.startswith("chr") else c - return int(c) if c.isdigit() else 99 - - -def overlaps(seg, others): - return [o for o in others if o[0] == seg[0] and o[1] < seg[2] and seg[1] < o[2]] - - -def read_genes(path): - """Protein-coding genes of a GENCODE gene GTF: chrom -> [(start, end, name)].""" - genes = {} - if not path: - return genes - with opener(path) as f: - for line in f: - if line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 9 or c[2] != "gene": - continue - if 'gene_type "protein_coding"' not in c[8]: - continue - m = re.search(r'gene_name "([^"]+)"', c[8]) - if m: - genes.setdefault(c[0], []).append((int(c[3]), int(c[4]), m.group(1))) - return genes - - -def genes_in(genes, chrom, start, end): - """The names of the genes overlapping [start, end), once each, in order. - IBDmix's start is the segment's first site and its end the next site, past - the segment (its README, --inclusive-end); a GTF gene is [s, e], 1-based.""" - name = chrom if str(chrom).startswith("chr") else f"chr{chrom}" - seen = [] - for s, e, n in sorted(genes.get(name, [])): - if s < end and e >= start and n not in seen: - seen.append(n) - return seen - - -def classify(per_archaic): - """Neanderthal: the union of the Altai and Vindija segments. Denisovan: the - Denisova segments that overlap no Neanderthal one (a stretch the Denisova - genome and a Neanderthal both match is counted as Neanderthal, the more - common source outside Oceania). Returns a list of - (chrom, start, end, ancestry, matched, lod).""" - nea = merge([(c, s, e, l, {n}) for n in NEANDERTHALS for c, s, e, l in per_archaic.get(n, [])]) - den = merge([(c, s, e, l, {"Denisova"}) for c, s, e, l in per_archaic.get("Denisova", [])]) - rows = [] - for c, s, e, l, who in nea: - also = {"Denisova"} if overlaps((c, s, e), den) else set() - rows.append((c, s, e, "Neanderthal", sorted(who) + sorted(also), l)) - for c, s, e, l, who in den: - if not overlaps((c, s, e), nea): - rows.append((c, s, e, "Denisovan", ["Denisova"], l)) - rows.sort(key=lambda r: (chrom_key(r[0]), r[1])) - return rows - - -def summarise(rows, spans_bp): - out = {} - for anc in ("Neanderthal", "Denisovan"): - r = [x for x in rows if x[3] == anc] - bp = sum(x[2] - x[1] for x in r) - key = anc.lower() - out[f"{key}_segments"] = len(r) - out[f"{key}_bp"] = bp - out[f"{key}_percent"] = f"{100.0 * bp / spans_bp:.2f}" if spans_bp else "0.00" - out[f"{key}_longest_bp"] = max((x[2] - x[1] for x in r), default=0) - return out - - -def cmd_report(a): - spans = [] - with open(a.spans) as f: - head = f.readline().rstrip("\n").split("\t") - if head[:4] != ["chrom", "first", "last", "sites"]: - die(f"{a.spans} is not the spans table of the genotypes step") - for line in f: - c = line.rstrip("\n").split("\t") - spans.append((c[0], int(c[1]), int(c[2]), int(c[3]))) - if not spans: - die(f"{a.spans} lists no chromosome") - spans_bp = sum(last - first + 1 for _, first, last, _ in spans) - per = {n: [] for n in ARCHAICS} - for chrom, *_ in spans: - for n in ARCHAICS: - p = os.path.join(a.ibdmix_dir, f"{chrom}.{n}.txt") - if not os.path.isfile(p): - die(f"no IBDmix output {p}") - per[n] += [(f"chr{c}" if not str(c).startswith("chr") else c, s, e, l) - for c, s, e, l in keep(read_segments(p))] - rows = classify(per) - genes = read_genes(a.genes) - summary = summarise(rows, spans_bp) - os.makedirs(a.out_dir, exist_ok=True) - seg_path = os.path.join(a.out_dir, f"{a.sample}_archaic_segments.tsv") - with open(seg_path, "w") as f: - f.write("\t".join(SEGMENT_HEADER) + "\n") - for c, s, e, anc, who, lod in rows: - g = genes_in(genes, c, s, e) if a.genes else [] - f.write(f"{c}\t{s}\t{e}\t{e - s}\t{anc}\t{','.join(who)}\t{lod:.1f}\t{','.join(g) if g else '.'}\n") - longest = sorted(rows, key=lambda r: (-(r[2] - r[1]), chrom_key(r[0]), r[1]))[:TOP] - meta = { - "sample": a.sample, - "method": "IBDmix", - "archaic_genomes": "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan", - "population": a.pop, - "min_lod": f"{MIN_LOD:g}", - "min_length_bp": MIN_LENGTH, - "analysed_bp": spans_bp, - "analysed_chromosomes": len(spans), - "analysed_sites": sum(s[3] for s in spans), - "genes": "GENCODE protein-coding" if a.genes else "none (no gene annotation installed)", - **summary, - } - with open(os.path.join(a.out_dir, f"{a.sample}_archaic.tsv"), "w") as f: - f.write("key\tvalue\n") - for k, v in meta.items(): - f.write(f"{k}\t{v}\n") - for i, (c, s, e, anc, who, lod) in enumerate(longest, 1): - g = genes_in(genes, c, s, e) if a.genes else [] - f.write(f"longest_{i}\t{c}:{s}-{e} {anc} {(e - s) / 1e6:.2f} Mb " - f"({'/'.join(who)}); genes: {', '.join(g) if g else 'none'}\n") - lines = [f"Archaic ancestry of {a.sample} (IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan)", - f" Analysed: {len(spans)} autosomes, {spans_bp / 1e6:.1f} Mb, {meta['analysed_sites']} sites; " - f"modern frequencies: 1000 Genomes {a.pop}", - f" Neanderthal: {summary['neanderthal_percent']}% ({summary['neanderthal_bp'] / 1e6:.1f} Mb in " - f"{summary['neanderthal_segments']} segments)", - f" Denisovan: {summary['denisovan_percent']}% ({summary['denisovan_bp'] / 1e6:.1f} Mb in " - f"{summary['denisovan_segments']} segments)"] - if longest: - lines.append(f" The {len(longest)} longest segments:") - for c, s, e, anc, who, lod in longest: - g = genes_in(genes, c, s, e) if a.genes else [] - lines.append(f" {c}:{s}-{e} {(e - s) / 1e6:.2f} Mb {anc} ({'/'.join(who)}, LOD {lod:.0f}) " - f"genes: {', '.join(g) if g else 'none'}") - lines.append(f" {CAVEAT}") - lines.append(f" Every segment: {os.path.basename(seg_path)}") - with open(os.path.join(a.out_dir, f"{a.sample}_archaic.txt"), "w") as f: - f.write("\n".join(lines) + "\n") - print("\n".join(lines)) - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - sub = p.add_subparsers(dest="cmd", required=True) - t = sub.add_parser("table", help="one GRCh37 chromosome of the archaic table (setup.sh --archaic)") - t.add_argument("--chrom", required=True) - t.add_argument("--modern", required=True) - t.add_argument("--archaic", action="append", default=[], help="NAME=FILE, once each for Altai, Vindija, Denisova") - t.add_argument("--mask", action="append", default=[], - help="NAME=BED: read that archaic genome only inside this mask (its MPI-EVA FilterBed)") - t.add_argument("--exclude", action="append", default=[], - help="BED: leave out every site inside these regions (segmental duplications, indel neighbourhoods)") - t.add_argument("--out", required=True) - m = sub.add_parser("strict", help="mark the rows outside the GRCh38 strict mask masked (setup.sh --archaic)") - m.add_argument("--chrom", required=True) - m.add_argument("--strict", required=True, help="BED: the 1000 Genomes GRCh38 strict mask") - m.add_argument("--rows", required=True, help="the lifted rows, tab-separated, no header") - m.add_argument("--out", required=True) - g = sub.add_parser("genotypes", help="IBDmix's genotype files for one sample (step 43)") - g.add_argument("--table-dir", required=True) - g.add_argument("--vcf", required=True) - g.add_argument("--pop", default="ALL") - g.add_argument("--pseudo", type=int, default=PSEUDO) - g.add_argument("--out-dir", required=True) - r = sub.add_parser("report", help="segments, shares and the ten longest (step 43)") - r.add_argument("--sample", required=True) - r.add_argument("--ibdmix-dir", required=True) - r.add_argument("--spans", required=True) - r.add_argument("--genes", default="") - r.add_argument("--pop", default="ALL") - r.add_argument("--out-dir", required=True) - a = p.parse_args(argv) - if a.cmd == "genotypes" and a.pseudo < 1: - die("--pseudo must be 1 or more", 2) - {"table": cmd_table, "strict": cmd_strict, "genotypes": cmd_genotypes, "report": cmd_report}[a.cmd](a) - - -if __name__ == "__main__": - main() diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 054d9a684..5aa4fd6eb 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -2169,7 +2169,7 @@ def sec_bloodtype(d, s): def sec_archaic(d, s): - """Step 43's summary (archaic/_archaic.tsv, bin/archaic_report.py): the + """Step 43's summary (archaic/_archaic.tsv, og archaic report): the Neanderthal and Denisovan shares and segment counts, the analysed span and the ten longest segments.""" p = first_existing(d, [f"archaic/{s}_archaic.tsv"]) diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs index 612a71212..6a3b6fc52 100644 --- a/rust/crates/og-ancestry/src/ancient.rs +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -1529,6 +1529,9 @@ mod tests { assert_eq!(year_label(1950.0), "1 BCE"); assert_eq!(year_label(1949.0), "1 CE"); assert_eq!(year_label(1750.5), "200 CE"); + assert_eq!(year_label(8025.0), "6075 BCE"); + assert_eq!(year_label(450.0), "1500 CE"); + assert_eq!(range_label(3000.0, 3000.0), "1050 BCE"); assert_eq!(range_label(7250.0, 6850.0), "5300-4900 BCE"); assert_eq!(range_label(1750.0, 1500.0), "200-450 CE"); assert_eq!(range_label(2250.0, 1750.0), "300 BCE-200 CE"); diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 37bdf7624..3d1b4e811 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -260,7 +260,6 @@ impl Stream { /// SNVs are skipped. fn next(&mut self) -> Out> { loop { - let path = self.text.path.display().to_string(); let Some(raw) = self.text.raw()? else { return Ok(None); }; @@ -268,10 +267,10 @@ impl Stream { let c: Vec<&str> = line.split('\t').collect(); if self.src == 0 { if c.len() < 9 { + let (n, head) = (c.len(), head80(raw)); return die(format!( - "{path}: a line has {} columns, want 9: {}", - c.len(), - head80(raw) + "{}: a line has {n} columns, want 9: {head}", + self.text.path.display() )); } if c[1].chars().count() != 1 || c[2].chars().count() != 1 { @@ -288,10 +287,10 @@ impl Stream { ))); } if c.len() < 4 { + let (n, head) = (c.len(), head80(raw)); return die(format!( - "{path}: a line has {} columns, want 4: {}", - c.len(), - head80(raw) + "{}: a line has {n} columns, want 4: {head}", + self.text.path.display() )); } let pos = need_int(c[0])?; @@ -732,7 +731,6 @@ struct Vcf { impl Vcf { fn next(&mut self) -> Out> { loop { - let path = self.text.path.display().to_string(); let Some(raw) = self.text.raw()? else { return Ok(None); }; @@ -743,10 +741,11 @@ impl Vcf { let c: Vec<&str> = line.split('\t').collect(); if raw.starts_with('#') { if c.len() != 10 { + let n = c.len().saturating_sub(9); return die(format!( - "{path} holds {} samples; step 43 reads a one-sample VCF \ + "{} holds {n} samples; step 43 reads a one-sample VCF \ (keep one with bcftools view -s ).", - c.len().saturating_sub(9) + self.text.path.display() )); } continue; diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f76068dcf..e2f0a3f24 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -47,11 +47,11 @@ alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/ancestry_fine_report.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/ancient_report.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -archaic-ancestry-1 assets/pgs_scores.tsv bin/archaic_report.py bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh bash-step-08-hla - bash-step-09-str assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh bash-step-10-telomere scripts/10-telomere-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 49bf9e2a5..645b5a3e4 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -1198,42 +1198,53 @@ def archaic(d, rng): def ancient(d, rng): - """Step 44: bin/ancient_report.py's nearest-group table, info table and - PC plot (its own svg_plot), made-up distances and PCs over a few - AADR-style group labels.""" - import ancient_report - groups = [("England_BellBeaker", "2450-2000 BCE", "United Kingdom", "Amesbury Down"), - ("Germany_CordedWare", "2900-2350 BCE", "Germany", "Esperstedt"), - ("Spain_BellBeaker", "2500-2000 BCE", "Spain", "Camino de las Yeseras"), - ("Czechia_EBA_Unetice", "2200-1700 BCE", "Czech Republic", "Moravia"), - ("Hungary_LBA", "1500-1100 BCE", "Hungary", "Tiszafüred"), - ("Italy_Imperial_Rome", "1-300 CE", "Italy", "Rome"), - ("Sweden_Viking", "800-1050 CE", "Sweden", "Sigtuna"), - ("France_Neolithic", "4500-3000 BCE", "France", "Gurgy"), - ("Turkey_Neolithic", "6500-6000 BCE", "Turkey", "Barcın"), - ("Russia_Yamnaya_Samara", "3300-2700 BCE", "Russia", "Samara")] + """Step 44: what og ancient report writes (the nearest-group table, the + info table, the text and the PC plot) for a made-up panel: present-day + people around the origin, ancient people of a few AADR-style group labels + each around its own point, and the person among them.""" + groups = [("England_BellBeaker", 4200, "United Kingdom", "Amesbury Down"), + ("Germany_CordedWare", 4600, "Germany", "Esperstedt"), + ("Spain_BellBeaker", 4200, "Spain", "Camino de las Yeseras"), + ("Czechia_EBA_Unetice", 3900, "Czech Republic", "Moravia"), + ("Hungary_LBA", 3250, "Hungary", "Tiszafüred"), + ("Italy_Imperial_Rome", 1800, "Italy", "Rome"), + ("Sweden_Viking", 1030, "Sweden", "Sigtuna"), + ("France_Neolithic", 5700, "France", "Gurgy"), + ("Turkey_Neolithic", 8200, "Turkey", "Barcın"), + ("Russia_Yamnaya_Samara", 4950, "Russia", "Samara")] person = (0.01234, -0.00456) - dist, nearest = 0.0, [] - lines = ["rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"] - for i, (g, dates, country, loc) in enumerate(groups, 1): - dist += rng.uniform(0.002, 0.012) - pc1, pc2 = person[0] + rng.uniform(-dist, dist), person[1] + rng.uniform(-dist, dist) - nearest.append({"group": g, "pc1": pc1, "pc2": pc2}) - lines.append(f"{i}\t{g}\t{rng.randint(3, 40)}\t{dist:.5f}\t{pc1:.5f}\t" - f"{pc2:.5f}\t{rng.randint(1000, 8500)}\t{dates}\t{country}\t{loc}\n") - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient.tsv"), "".join(lines)) - panel = {f"m{i}": ("present", (rng.gauss(0.0, 0.03), rng.gauss(0.0, 0.025))) for i in range(300)} - ancients = {} - for i in range(200): - panel[f"a{i}"] = (ancient_report.ANCIENT_LABEL, (rng.gauss(0.005, 0.035), rng.gauss(0.0, 0.03))) - ancients[f"a{i}"] = {"date_bp": str(rng.randint(500, 9000))} - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_pca.svg"), - ancient_report.svg_plot(person, panel, ancients, SAMPLE, nearest)) - info = [("release", "v62.0"), ("sites_used", rng.randint(520_000, 560_000)), ("panel_sites", 597_573), - ("present_day_people", 819), ("ancient_people", 4_210), ("ancient_groups", 612), - ("pcs_in_distance", 4), ("pc1", f"{person[0]:.5f}"), ("pc2", f"{person[1]:.5f}")] - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_info.tsv"), - "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in info)) + evals = "#eigvals: 9.000 4.000 2.000 1.000\n" + moderns = [f"m{i} " + " ".join(f"{rng.gauss(0.0, sd):.6f}" for sd in (0.03, 0.025, 0.02, 0.015)) + + f" {('French', 'Sardinian', 'Basque')[i % 3]}\n" for i in range(300)] + ancients, anc_rows = [], ["id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"] + for i, (g, bp, country, loc) in enumerate(groups): + spread = 0.004 + 0.006 * i + cx, cy = person[0] + rng.uniform(-spread, spread), person[1] + rng.uniform(-spread, spread) + for j in range(rng.randint(3, 12)): + aid = f"a{i}_{j}" + ancients.append(f"{aid} {cx + rng.gauss(0, 0.004):.6f} {cy + rng.gauss(0, 0.004):.6f} 0.000000 0.000000 Ancient\n") + date = bp + rng.randint(-150, 150) + anc_rows.append(f"{aid}\t{g}\t{date}\t50\t.\t{loc}\t{country}\t50\t10\t500000\n") + with tempfile.TemporaryDirectory() as work: + paths = {n: os.path.join(work, n) for n in ("panel.evec", "run.evec", "ancients.tsv", "stats.tsv")} + put(paths["panel.evec"], evals + "".join(moderns + ancients)) + put(paths["run.evec"], evals + "".join(moderns) + + f"{SAMPLE} {person[0]:.6f} {person[1]:.6f} 0.000000 0.000000 Person\n") + put(paths["ancients.tsv"], "".join(anc_rows)) + put(paths["stats.tsv"], f"panel_sites\t597573\nsites_genotyped\t560000\nother_allele\t120\n" + f"sites_used\t{rng.randint(520_000, 560_000)}\n") + out = os.path.join(d, "ancient") + os.makedirs(out, exist_ok=True) + r = subprocess.run([og_binary(), "ancient", "report", "--panel-evec", paths["panel.evec"], + "--run-evec", paths["run.evec"], "--ancients", paths["ancients.tsv"], + "--sample", SAMPLE, "--release", "v62.0", "--stats", paths["stats.tsv"], + "--out-tsv", os.path.join(out, f"{SAMPLE}_ancient.tsv"), + "--out-info", os.path.join(out, f"{SAMPLE}_ancient_info.tsv"), + "--out-txt", os.path.join(out, f"{SAMPLE}_ancient.txt"), + "--out-svg", os.path.join(out, f"{SAMPLE}_ancient_pca.svg")], + stdout=subprocess.DEVNULL) + if r.returncode: + sys.exit("ERROR: og ancient report failed on the made-up panel") def exomiser(d): diff --git a/tests/e2e/ancient-similarity-1.sh b/tests/e2e/ancient-similarity-1.sh index c1b0e8e7f..21941d932 100755 --- a/tests/e2e/ancient-similarity-1.sh +++ b/tests/e2e/ancient-similarity-1.sh @@ -46,7 +46,7 @@ for e in ANNO:anno IND:ind SNP:snp; do curl -fsSL --retry 3 -o "${REAL}/aadr.${e#*:}" "https://dataverse.harvard.edu/api/access/datafile/${!id_var}" check_eq "AADR ${AADR_VERSION} .${e#*:}: the md5 versions.env pins" "$(md5sum "${REAL}/aadr.${e#*:}" | cut -d' ' -f1)" "${!md5_var}" done -python3 "${REPO}/bin/ancient_report.py" select --anno "${REAL}/aadr.anno" --ind "${REAL}/aadr.ind" \ +"${REPO}/bin/og" ancient select --anno "${REAL}/aadr.anno" --ind "${REAL}/aadr.ind" \ --out-ind "${REAL}/panel.ind" --poplist "${REAL}/poplist.txt" --ancients "${REAL}/ancients.tsv" | tee "${CASE_TMP}/select.log" check_ge "real AADR: present-day reference people" "$(awk '$3 != "Ignore" && $3 != "Ancient"' "${REAL}/panel.ind" | wc -l | tr -d ' ')" 800 check_ge "real AADR: present-day groups" "$(wc -l < "${REAL}/poplist.txt" | tr -d ' ')" 35 diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py index f91218a6d..b0e7efee9 100644 --- a/tests/test_ancient_report.py +++ b/tests/test_ancient_report.py @@ -1,34 +1,35 @@ #!/usr/bin/env python3 -"""bin/ancient_report.py (step 44) on synthetic files. +"""og ancient (step 44) on synthetic files, run as the pipeline runs it. Checks: - 1. dates: years before 1950 become BCE and CE the way the AADR writes - them (4250 BP is the middle of its 2600-2000 BCE; 1950 BP is 1 BCE, - 1949 BP is 1 CE), and a population's span is written oldest first, with - one era when both ends share it; - 2. ranking: on a synthetic .evec the nearest ancient group is the one whose + 1. ranking: on a synthetic .evec the nearest ancient group is the one whose mean position is closest on the weighted PCs, a group far on PC1 but with a small eigenvalue on PC3 is ranked by the weighted distance, and - each row carries its dates, country and member count; - 3. axes: a run whose PC2 has the opposite sign is turned back before the + each row carries its dates (oldest first, AADR style BCE), country and + member count; + 2. axes: a run whose PC2 has the opposite sign is turned back before the person is placed, and a run whose present-day coordinates differ stops with the reason instead of placing the person on other axes; - 4. liftover: a site inside a block maps with the block's offset, a site on - a minus-strand chain maps to the mirrored position with its alleles - complemented, a site in a gap or covered by two chains is dropped; - 5. genotypes: a bcftools genotype gives the copies of the .snp file's first - allele, phased or not; a half call, an indel and an allele that is - neither of the two are missing; person-geno appends one column, 9 where - the person has no genotype; - 6. select: the present-day reference people come from MODERN_GROUPS, the + 3. liftover: lift keeps a site inside a block, a site on a minus-strand + chain with its alleles complemented, and drops a site in a gap, one + whose alleles miss the GRCh38 base, chrX and a monomorphic marker; + 4. genotypes: person-geno appends one column, 9 where the person has no + usable genotype, and stops on a genotype file shorter than the .snp file; + 5. select: the present-day reference people come from MODERN_GROUPS, the ancient ones need a date, a passing assessment, enough sites and a place in West Eurasia, one genetic id per individual; - 7. the report command writes the table, the summary, the text and an SVG + 6. the report command writes the table, the summary, the text and an SVG with the person marked; - 8. bin/render_report.py shows them: an HTML card with the table and the + 7. bin/render_report.py shows them: an HTML card with the table and the plot, and a section of the text report. -Run: python3 tests/test_ancient_report.py +The Rust unit tests (rust/crates/og-ancestry, og-intervals chain) check the +date labels, the dosages and the single-position lift in detail, and +tests/parity checks each command against the Python it replaced. Without an +og binary the test is skipped with a note; with OG_REQUIRED set the skip is a +failure. + +Run: OG=bin/og python3 tests/test_ancient_report.py """ import os import shutil @@ -37,11 +38,13 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import ancient_report as ar # noqa: E402 - -SCRIPT = os.path.join(REPO, "bin", "ancient_report.py") +OG = os.environ.get("OG", "") FAILS = 0 +CAVEAT = "Closeness on a PCA is genetic similarity, not descent" + +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) def check(desc, ok, detail=""): @@ -57,17 +60,11 @@ def write(path, text): return path -# --- 1. dates --------------------------------------------------------------------- -check("4250 BP is 2300 BCE, the AADR's midpoint of 2600-2000 BCE", ar.year_label(4250) == "2300 BCE", ar.year_label(4250)) -check("8025 BP is 6075 BCE", ar.year_label(8025) == "6075 BCE", ar.year_label(8025)) -check("1950 BP is 1 BCE", ar.year_label(1950) == "1 BCE", ar.year_label(1950)) -check("1949 BP is 1 CE", ar.year_label(1949) == "1 CE", ar.year_label(1949)) -check("450 BP is 1500 CE", ar.year_label(450) == "1500 CE", ar.year_label(450)) -check("a span in one era is written once", ar.range_label(7250, 6850) == "5300-4900 BCE", ar.range_label(7250, 6850)) -check("a span across the eras names both", ar.range_label(2250, 1750) == "300 BCE-200 CE", ar.range_label(2250, 1750)) -check("a span of one date is that date", ar.range_label(3000, 3000) == "1050 BCE", ar.range_label(3000, 3000)) +def og(*args): + return subprocess.run([OG, "ancient", *args], capture_output=True, text=True) + -# --- 2. ranking ------------------------------------------------------------------- +# --- 1. ranking ------------------------------------------------------------------- # Present-day people M0..M19 build the axes; three ancient groups. EVALS = [9.0, 4.0, 0.01, 0.01] @@ -94,36 +91,62 @@ def evec(rows, evals=EVALS): "B1": {"id": "B1", "group": "Anatolia_N", "date_bp": "8200", "country": "Turkey", "locality": "Barcin"}, "C1": {"id": "C1", "group": "Odd_C", "date_bp": "3000", "country": "Italy", "locality": "Somewhere"}, } -person = [0.08, 0.09, 0.0, 0.0] -rows, n = ar.nearest_populations(person, PANEL, ANCIENTS, EVALS, 4) -check("three ancient groups ranked", n == 3 and len(rows) == 3, (n, rows)) -order = [r["group"] for r in rows] -check("the nearest group is first (Steppe_EBA)", order[0] == "Steppe_EBA", order) -check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1] == "Odd_C", order) -check("the farthest group is last (Anatolia_N)", order[-1] == "Anatolia_N", order) -st = rows[0] -check("the row has the member count, dates and place", st["n"] == 2 and st["dates"] == "2850-2650 BCE" - and st["country"] == "Russia" and st["locality"] == "Samara", st) -check("ranks are 1..n", [r["rank"] for r in rows] == [1, 2, 3], rows) -unweighted = ar.nearest_populations(person, PANEL, ANCIENTS, [1.0, 1.0, 1.0, 1.0], 4)[0] -check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", - [r["group"] for r in unweighted] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], [r["group"] for r in unweighted]) +# The person's run: the present-day people with PC2 flipped, as smartpca may +# write it, and the person. +RUN = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} +RUN["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) -# --- 3. axes ---------------------------------------------------------------------- -run = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} -run["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) -signs = ar.axis_signs(PANEL, run, 2) -check("a flipped PC2 is found and turned back", signs == [1.0, -1.0], signs) -bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} -try: - ar.axis_signs(PANEL, bad, 2) - check("other axes stop the report", False, "no error") -except ar.Fail as e: - check("other axes stop the report, naming the PC", "PC1" in str(e), str(e)) - -# --- 4. liftover -------------------------------------------------------------------- tmp = tempfile.mkdtemp() try: + stats = write(os.path.join(tmp, "stats.tsv"), "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n") + at = os.path.join(tmp, "anc.tsv") + with open(at, "w") as f: + f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") + for r in ANCIENTS.values(): + f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") + outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} + + def report(panel_evec, run_evec, with_stats=True): + return og("report", "--panel-evec", panel_evec, "--run-evec", run_evec, "--ancients", at, + "--sample", "S", "--release", "test", *(["--stats", stats] if with_stats else []), + "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], + "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]) + + def table(): + with open(outs["ancient.tsv"]) as f: + head, *rows = [line.split("\t") for line in f.read().splitlines()] + return [dict(zip(head, r)) for r in rows] + + pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) + re_ = write(os.path.join(tmp, "run.evec"), evec(RUN)) + p = report(pe, re_) + check("report exits 0", p.returncode == 0, p.stderr) + rows = table() if p.returncode == 0 else [] + order = [r["group"] for r in rows] + check("three ancient groups ranked", len(rows) == 3, rows) + check("the nearest group is first (Steppe_EBA), from the flipped run turned back", + order[:1] == ["Steppe_EBA"], order) + check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1:2] == ["Odd_C"], order) + check("the farthest group is last (Anatolia_N)", order[-1:] == ["Anatolia_N"], order) + st = rows[0] if rows else {} + check("the row has the member count, dates and place", st.get("n") == "2" and st.get("dates") == "2850-2650 BCE" + and st.get("country") == "Russia" and st.get("locality") == "Samara", st) + check("ranks are 1..n", [r["rank"] for r in rows] == ["1", "2", "3"], rows) + flat_pe = write(os.path.join(tmp, "flat_panel.evec"), evec(PANEL, [1.0, 1.0, 1.0, 1.0])) + flat_re = write(os.path.join(tmp, "flat_run.evec"), evec(RUN, [1.0, 1.0, 1.0, 1.0])) + p = report(flat_pe, flat_re) + check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", + p.returncode == 0 and [r["group"] for r in table()] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], p.stderr) + + # --- 2. axes ---------------------------------------------------------------------- + p = og("align", "--panel-evec", pe, "--run-evec", re_, "--pcs", "2") + check("align: a flipped PC2 is found", p.returncode == 0 and "PC1+ PC2-" in p.stdout, (p.stdout, p.stderr)) + bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} + bad_re = write(os.path.join(tmp, "bad.evec"), evec(bad)) + p = og("align", "--panel-evec", pe, "--run-evec", bad_re, "--pcs", "2") + check("align: other axes stop, naming the PC", p.returncode == 1 and "PC1" in p.stderr, (p.returncode, p.stderr)) + + # --- 3. liftover -------------------------------------------------------------------- chain = write(os.path.join(tmp, "test.chain"), "\n".join([ # chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then 1150-1200 -> 2150-2200 "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", @@ -134,19 +157,6 @@ def evec(rows, evals=EVALS): # chr3: two chains over 100-200 "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", "100", "", "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", "100", ""]) + "\n") - ix = ar.index_blocks(ar.read_chain(chain)) - check("a site in the first block moves by its offset", ar.lift_point(ix, "chr1", 1051) == ("chr1", 2051, "+"), - ar.lift_point(ix, "chr1", 1051)) - check("a site in the second block moves by its offset", ar.lift_point(ix, "chr1", 1160) == ("chr1", 2160, "+"), - ar.lift_point(ix, "chr1", 1160)) - check("a site in the gap is dropped", ar.lift_point(ix, "chr1", 1120) is None, ar.lift_point(ix, "chr1", 1120)) - # 0-based 510 -> reverse 310 -> forward 10000 - 1 - 310 = 9689 (0-based) = 9690 1-based - check("a minus-strand site is mirrored", ar.lift_point(ix, "chr2", 511) == ("chr2", 9690, "-"), - ar.lift_point(ix, "chr2", 511)) - check("a site two chains cover is dropped", ar.lift_point(ix, "chr3", 171) is None, ar.lift_point(ix, "chr3", 171)) - check("a site one of them covers maps", ar.lift_point(ix, "chr3", 121) == ("chr3", 121, "+"), - ar.lift_point(ix, "chr3", 121)) - # The lift command against a small FASTA. seq1 = ["N"] * 3000 seq1[2050] = "G" # 1-based 2051 seq1[2159] = "T" # 1-based 2160 @@ -174,38 +184,32 @@ def evec(rows, evals=EVALS): "rs7 1 0.0 1051 G X", # monomorphic marker: dropped ]) + "\n") out = os.path.join(tmp, "sites.tsv") - rc = ar.main(["lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out]) + p = og("lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out) got = open(out).read().splitlines() if os.path.exists(out) else [] check("lift keeps rs1, rs2 and the complemented rs4, in GRCh38 order", - rc == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", - "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (rc, got)) + p.returncode == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", + "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (p.returncode, got, p.stderr)) - # --- 5. genotypes ----------------------------------------------------------------- - check("A/A is two copies of allele1 A", ar.dosage("A/A", "A", "G") == 2) - check("A|G is one copy", ar.dosage("A|G", "A", "G") == 1) - check("G/G is none", ar.dosage("G/G", "A", "G") == 0) - check("a half call is missing", ar.dosage("A", "A", "G") is None) - check("an allele outside the pair is missing", ar.dosage("A/T", "A", "G") is None) - check("an indel is missing", ar.dosage("A/AT", "A", "G") is None) + # --- 4. genotypes ----------------------------------------------------------------- msnp = write(os.path.join(tmp, "moderns.snp"), "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n") mgeno = write(os.path.join(tmp, "moderns.geno"), "012\n220\n999\n101\n") gts = write(os.path.join(tmp, "gts.tsv"), "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n") merged = os.path.join(tmp, "merged.geno") - stats = os.path.join(tmp, "stats.tsv") - rc = ar.main(["person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, - "--out", merged, "--stats", stats]) + pstats = os.path.join(tmp, "pstats.tsv") + p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, + "--out", merged, "--stats", pstats) got = open(merged).read().splitlines() if os.path.exists(merged) else [] check("person-geno appends one column: 1, 0, 9 for X, 9 for an allele outside the pair", - rc == 0 and got == ["0121", "2200", "9999", "1019"], (rc, got)) - st = dict(l.split("\t") for l in open(stats).read().splitlines()) if os.path.exists(stats) else {} + p.returncode == 0 and got == ["0121", "2200", "9999", "1019"], (p.returncode, got, p.stderr)) + st = dict(line.split("\t") for line in open(pstats).read().splitlines()) if os.path.exists(pstats) else {} check("its counts", st == {"panel_sites": "4", "sites_genotyped": "3", "other_allele": "1", "sites_used": "2"}, st) short = write(os.path.join(tmp, "short.geno"), "012\n220\n") - rc = ar.main(["person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, - "--out", merged, "--stats", stats]) - check("a genotype file shorter than the .snp file stops", rc == 1, rc) + p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, + "--out", merged, "--stats", pstats) + check("a genotype file shorter than the .snp file stops", p.returncode == 1, (p.returncode, p.stderr)) - # --- 6. select ------------------------------------------------------------------------ + # --- 5. select ------------------------------------------------------------------------ head = ["Genetic ID (long text)", "Persistent Genetic ID", "Individual ID", '"Date mean in BP in years before 1950"', "Date standard deviation in BP [x]", "Full Date One of two formats", "Group ID", "Locality", "Political Entity", "Latitude", "Longitude", "Data type", @@ -231,43 +235,32 @@ def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): anno = write(os.path.join(tmp, "x.anno"), "\n".join("\t".join(r) for r in [head] + rows_) + "\n") ind = write(os.path.join(tmp, "x.ind"), "".join(f"{r[0]} U {r[6]}\n" for r in rows_)) oind, plist, anc = (os.path.join(tmp, n) for n in ("panel.ind", "poplist.txt", "ancients.tsv")) - rc = ar.main(["select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc]) - labels = dict(l.split("\t")[0::2] for l in open(oind).read().splitlines()) if rc == 0 else {} + p = og("select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc) + rc = p.returncode + labels = dict(line.split("\t")[0::2] for line in open(oind).read().splitlines()) if rc == 0 else {} check("select: the 60 French shotgun genomes are the reference, one id per person (F0.DG)", - rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", labels) + rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", + (labels, p.stderr)) check("select: Yakut and a Human Origins genome are ignored", labels.get("Y1.DG") == "Ignore" and labels.get("H1.HO") == "Ignore", labels) check("select: only A1 is projected", [k for k, v in labels.items() if v == "Ancient"] == ["A1.AG"], labels) check("select: the population list", open(plist).read() == "French\n" if rc == 0 else False) check("select: ancients.tsv holds A1 with its group", - rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), open(anc).read() if rc == 0 else "") + rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), + open(anc).read() if rc == 0 else "") - # --- 7. the report command ---------------------------------------------------------- - pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) - re_ = write(os.path.join(tmp, "run.evec"), evec(run)) - at = os.path.join(tmp, "anc.tsv") - with open(at, "w") as f: - f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for r in ANCIENTS.values(): - f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") - outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} - p = subprocess.run([sys.executable, SCRIPT, "report", "--panel-evec", pe, "--run-evec", re_, "--ancients", at, - "--sample", "S", "--release", "test", "--stats", stats, - "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], - "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]], capture_output=True, text=True) + # --- 6. the report command ---------------------------------------------------------- + p = report(pe, re_) check("report exits 0", p.returncode == 0, p.stderr) - tsv = open(outs["ancient.tsv"]).read().splitlines() if p.returncode == 0 else [] - check("report: the table's first row is Steppe_EBA, from the flipped run turned back", - len(tsv) == 4 and tsv[1].split("\t")[1] == "Steppe_EBA", tsv) - info = dict(l.split("\t") for l in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} + info = dict(line.split("\t") for line in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} check("report: the person's PC2 in the panel's sign", info.get("pc2") == "0.09000", info) check("report: the sites used", info.get("sites_used") == "2", info) svg = open(outs["pca.svg"]).read() if p.returncode == 0 else "" check("report: the SVG is an SVG with a dark background and the person's label", svg.startswith("S" in svg and svg.rstrip().endswith(""), svg[:200]) txt = open(outs["ancient.txt"]).read() if p.returncode == 0 else "" - check("report: the text carries the caveat and says qpAdm is not run", ar.CAVEAT in txt and "qpAdm" in txt, txt) - # --- 8. both reports read the outputs ---------------------------------------------- + check("report: the text carries the caveat and says qpAdm is not run", CAVEAT in txt and "qpAdm" in txt, txt) + # --- 7. both reports read the outputs ---------------------------------------------- sd = os.path.join(tmp, "S") os.makedirs(os.path.join(sd, "ancient")) for k, f in outs.items(): @@ -284,10 +277,7 @@ def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): check("the text report has the section with the nearest group and the caveat", "Ancient Populations (AADR PCA, step 44)" in text and "1. Steppe_EBA (" in text and "not descent" in text, text[-600:]) - p = subprocess.run([sys.executable, SCRIPT, "report", "--panel-evec", pe, "--run-evec", pe, "--ancients", at, - "--sample", "S", "--release", "test", "--out-tsv", outs["ancient.tsv"], "--out-info", - outs["info.tsv"], "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]], - capture_output=True, text=True) + p = report(pe, pe, with_stats=False) check("report: a run without the person stops", p.returncode == 1 and "no row for S" in p.stderr, p.stderr) finally: shutil.rmtree(tmp) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index a54d508e9..844658cd4 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/archaic_report.py (step 43) on a synthetic archaic panel, with the real IBDmix image. +"""og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. Checks: 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic @@ -37,7 +37,12 @@ Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; CI never sets it. -Run: python3 tests/test_archaic_report.py +The Rust unit tests (rust/crates/og-ancestry) check pick_call's symmetry and +which genes touch a segment's ends, and tests/parity checks each command +against the Python it replaced. Without an og binary the test is skipped with +a note; with OG_REQUIRED set the skip is a failure. + +Run: OG=bin/og python3 tests/test_archaic_report.py """ import gzip import os @@ -49,10 +54,17 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import archaic_report as ar # noqa: E402 - -SCRIPT = os.path.join(REPO, "bin", "archaic_report.py") +OG = os.environ.get("OG", "") FAILS = 0 +TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", + "AF_ALL", "AF_AFR", "AF_AMR", "AF_EAS", "AF_EUR", "AF_SAS"] +SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] +MASKED = 10 +ARCHAICS = ("Altai", "Vindija", "Denisova") + +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) def check(desc, ok, detail=""): @@ -68,7 +80,7 @@ def write(path, text): def run(*args): - return subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) + return subprocess.run([OG, "archaic", *args], capture_output=True, text=True) def image(var): @@ -166,25 +178,30 @@ def table_info(out, extra=()): r.returncode == 1 and "no interval" in r.stderr, r.stderr) # --- 1b. strict: rows outside the GRCh38 strict mask are marked, not dropped ------- -# Lifted rows (13 columns, no header). Strict mask 1-based 101-300 on chr22: -# 100 is outside, 150 and 300 inside, 301 outside. A duplicate position keeps -# the first row. Codes 0, 1, 2 outside become 10, 11, 12; 9 stays 9; a code -# already masked is not marked twice. +# Lifted rows (13 columns, no header), sorted by position as Picard writes +# them. Strict mask 1-based 101-300 on chr22: 100 is outside, 150 and 300 +# inside, 301 outside. A duplicate position keeps the first row. Codes 0, 1, 2 +# outside become 10, 11, 12; 9 stays 9; a code already masked is not marked +# twice. rows = ["chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", + "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0", - "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0"] + "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0"] write(f"{TMP}/lifted.tsv", "\n".join(rows) + "\n") write(f"{TMP}/strict.bed", "chr22\t100\t300\n") r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked.tsv") check("strict exits 0", r.returncode == 0, r.stderr) got = [l.rstrip("\n").split("\t") for l in open(f"{TMP}/marked.tsv")] by = {int(c[1]): c[4:7] for c in got} -check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 301, 300], [c[1] for c in got]) +check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 300, 301], [c[1] for c in got]) check("outside the strict mask 2, 0 and unknown become 12, 10 and 9", by[100] == ["12", "10", "9"], by.get(100)) check("inside the strict mask the calls are untouched (last base 300 too)", by[150] == ["1", "1", "1"] and by[300] == ["0", "0", "0"], (by.get(150), by.get(300))) check("a masked call stays as it is, and the first of a duplicate wins", by[301] == ["10", "10", "12"], by.get(301)) +write(f"{TMP}/unsorted.tsv", "\n".join([rows[0], rows[3], rows[2]]) + "\n") +r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/unsorted.tsv", "--out", f"{TMP}/marked3.tsv") +check("rows out of position order stop the step (a duplicate is found only next to its twin)", + r.returncode == 2 and "sorted" in r.stderr and not os.path.exists(f"{TMP}/marked3.tsv"), (r.returncode, r.stderr)) write(f"{TMP}/strict_other.bed", "chr21\t0\t10\n") r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict_other.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked2.tsv") check("a strict mask with no interval on the chromosome stops the step", r.returncode != 0 and "no interval" in r.stderr, r.stderr) @@ -192,7 +209,7 @@ def table_info(out, extra=()): # --- 2. genotypes ------------------------------------------------------------------ tdir = f"{TMP}/table" os.makedirs(tdir) -hdr = "\t".join(ar.TABLE_HEADER) + "\n" +hdr = "\t".join(TABLE_HEADER) + "\n" rows = ["chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", @@ -267,12 +284,6 @@ def table_info(out, extra=()): check("a 0/0 half after the carried SNV keeps the SNV call", code3(6000) == "1", g3.get(6000)) check("a usable record of the same SNV wins over a filtered one", code3(7000) == "2", g3.get(7000)) check("two 0/0 halves are the reference (0)", code3(8000) == "0", g3.get(8000)) -pc_code = lambda r: 9 if r is ar.CLASH else r[2] -check("pick_call gives the same code in either order", - all(pc_code(ar.pick_call(x, y)) == pc_code(ar.pick_call(y, x)) - for x, y in [(("A", "G", 1), ("A", None, 9)), (("A", "G", 1), ("A", "T", 0)), (("A", "G", 9), ("A", "G", 2)), - (("A", "G", 0), ("A", "T", 0)), (("A", "G", 1), ("A", "T", 2)), (("A", "G", 9), ("A", "T", 0)), - (("A", "G", 1), ("A", "G", 2))])) # Masked calls: IBDmix gets the call itself and a mask per genome of the # positions where it is masked, the runs of written rows closed by the next @@ -291,7 +302,7 @@ def table_info(out, extra=()): check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", gm.get(100) == ["2", "0", "9"] and gm.get(200) == ["0", "1", "0"] and gm.get(600) == ["1", "0", "0"], gm) masks = {g: open(f"{TMP}/gtm/chr22.{g}.mask.bed").read() if os.path.exists(f"{TMP}/gtm/chr22.{g}.mask.bed") else None - for g in ar.ARCHAICS} + for g in ARCHAICS} check("Altai's mask: 100-200, then 400-600 (the left-out row 500 does not close it)", masks["Altai"] == "22\t99\t200\n22\t399\t600\n", masks["Altai"]) check("Vindija's mask: 200 alone", masks["Vindija"] == "22\t199\t200\n", masks["Vindija"]) @@ -321,7 +332,7 @@ def table_info(out, extra=()): "--out-dir", f"{TMP}/out") check("report exits 0", r.returncode == 0, r.stderr) segs = [l.split("\t") for l in open(f"{TMP}/out/S1_archaic_segments.tsv").read().splitlines()] -check("segments header", segs[0] == ar.SEGMENT_HEADER, segs[0]) +check("segments header", segs[0] == SEGMENT_HEADER, segs[0]) body = segs[1:] check("two segments: the merged Neanderthal one and the Denisovan one", len(body) == 2, body) check("Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge into Neanderthal 1.0-1.5 Mb", @@ -367,9 +378,6 @@ def table_info(out, extra=()): check("a row cut after its key is unreadable, not a crash", False) except cs.Unreadable: check("a row cut after its key is unreadable, not a crash", True) -genes = {"chr22": [(90, 100, "ENDS_AT_START"), (100, 120, "INSIDE"), (200, 210, "STARTS_AT_END"), (50, 99, "BEFORE")]} -check("genes: a segment [100, 200) takes a gene ending at its first site, not one starting at its end", - ar.genes_in(genes, "22", 100, 200) == ["ENDS_AT_START", "INSIDE"], ar.genes_in(genes, "22", 100, 200)) check("no file: the section is missing", cs.sec_archaic(f"{TMP}/nowhere", "S1") == (None, {})) # --- 4. the real IBDmix ------------------------------------------------------------- @@ -411,14 +419,14 @@ def synthetic(d, plant, mid=None): if mid and MID[0] <= pos < MID[1]: if kind == 0: g = [False, False] - ta, tv = (al + ar.MASKED, vi + ar.MASKED) if mid == "masked" else (9, 9) + ta, tv = (al + MASKED, vi + MASKED) if mid == "masked" else (9, 9) rows.append(f"chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t" + "\t".join([f"{f:.4f}"] * 6)) if plant and DEN[0] <= pos < DEN[1]: g[0] = de == 2 if any(g): calls.append(f"chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{'1/1' if all(g) else '0/1'}") with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(ar.TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") + f.write("\t".join(TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") From 796f4cd6681d4a72aa0a6c4393002e95352a576d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:21 +0100 Subject: [PATCH 04/86] docs(ancestry): steps 43 and 44 run og archaic and og ancient Bead: pgp-bs9u.21 --- CHANGELOG.md | 8 ++++++++ docs/43-archaic.md | 6 +++--- docs/44-ancient.md | 2 +- docs/rust.md | 6 +++--- rust/README.md | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f5f4625..7dfccbfb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 43's genotype files for 3 million table rows take 6 s and 159 MB instead of 12 s and 213 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: + - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. + - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. + - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. + - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. + - Malformed input stops with one `ERROR:` line, not a Python traceback. + - `sites_genotyped` in step 44's stats still counts a position twice when the AADR holds two SNPs there, as the Python did; the genotypes themselves are right. + - Step 37's own Y tools now run in the Rust `og` binary: `og y tree-positions` and `og y tree-place` (the placement on the current YFull YTree), `og y private-snps` (the private SNP list) and `og y yleaf-positions` (Yleaf's marker positions). They replace `bin/y_tree_place.py` and `bin/y_private_snps.py`, which are gone, and the `positions` command of `bin/yleaf_run.py`, which keeps only `predict`. og is static, so each part still runs in the Yleaf image. The three tables are the same, byte for byte, on the 43 parity cases, except where a line below says otherwise. On the pinned YTree 14.06.0 with a synthetic pileup of 362,055 positions, `og y tree-place` takes 0.8 s and 99 MB where the Python took 3.6 s and 389 MB; on a synthetic chrY VCF of 20,000 records, `og y private-snps` takes 0.3 s and 24 MB where the Python took 1.1 s and 330 MB. What changes on purpose: - `og y tree-place` reads the pileup by contig and position, and only its `chrY` (or `Y`) lines. The Python keyed it by position only, so a line on another contig could overwrite a chrY count. Lines on another contig are now counted at the end of the step line, and a pileup with both `chrY` and `Y` lines stops with exit 1. Step 37 makes a chrY-only pileup, so its results do not change. - `og y private-snps` refuses a callable mask with more than one FASTA record (exit 1). The Python joined the records into one sequence, which shifts every position after the first. The pinned mask has one record. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 25e33f93b..3e706e1ba 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -13,7 +13,7 @@ Most people outside Africa carry about 2% Neanderthal DNA, and people with ances ## Tool - [IBDmix](https://github.com/PrincetonUniversity/IBDmix) 1.0.1 (Chen et al. 2020, Cell 180:677, doi:10.1016/j.cell.2020.01.012) -- `bin/archaic_report.py` (standard library Python): IBDmix's genotype files, then the segments and the summary +- `og archaic` (OwnGenome's Rust binary, `docs/rust.md`): the table at setup, IBDmix's genotype files, then the segments and the summary IBDmix needs no unadmixed reference population. It scores each site from the modern allele frequency and the two genotypes, so it finds archaic segments in Africans too, where methods that use Africans as the unadmixed reference cannot. @@ -57,13 +57,13 @@ Chen et al. do not score every site. Each archaic genome comes with its own mask - a genome's call outside that genome's FilterBed is kept and marked masked in the table (10, 11 or 12: 10 plus its ALT count); - every call at a site inside a segmental duplication (UCSC's GRCh37 track), or inside the window of a 1000 Genomes indel, is marked masked the same way. The window is 5 bp on each side of the indel, reaching past the longer of its REF and ALT, as the published script writes it; - step 43 writes each masked call to IBDmix as its ALT count, and the masked positions to a mask file per chromosome and archaic genome, which it passes to IBDmix with `-r`; -- every call at a site outside the 1000 Genomes GRCh38 strict mask is marked masked the same way, in all three archaic columns. `setup.sh --archaic` does that after the liftover (`archaic_report.py strict`), because the mask is on GRCh38 and the table is built on GRCh37. +- every call at a site outside the 1000 Genomes GRCh38 strict mask is marked masked the same way, in all three archaic columns. `setup.sh --archaic` does that after the liftover (`og archaic strict`), because the mask is on GRCh38 and the table is built on GRCh37. The first version of the masks dropped the masked calls instead (code 9, or no row), and a second one still dropped every row outside the strict mask. IBDmix then had nothing to score at those sites, the mismatches included, so a segment could run through masked sequence that contradicts it. That is why HG002's share stayed above the published figure after the masks; see Runtime below. One of their exclusions is not applied here: the CpG mask, which needs a GRCh37 reference and the chimpanzee, orangutan and macaque SNP sets, none of which this pipeline installs. What that leaves out is written down in the Notes below. -The filters cut hard: on chr22 the strict mask covers 25.2 Mb of 50.8, the Vindija FilterBed 19.6 Mb, and segmental duplications take out 4.3 Mb. Without them the step called sequence from positions the archaic genomes cannot be read at, and HG002's Neanderthal total was higher still: the filters took it from 115.2 Mb to 107.5 Mb, which is still 2.1 times the published median (see the same-person section under Runtime). The filters are part of the data version: a change to a pin or to a mask means `snpad2016-1kgp3-v3` and a rebuild. `setup.sh --archaic` also rebuilds a table whose build code changed (its stamp holds the sha256 of the table half of `bin/archaic_report.py`), so a table built before the masked codes, or before the strict-mask rows were kept, is rebuilt on the next `setup.sh --archaic`. Until then step 43 reads it as before: a table with no masked code gives empty masks. +The filters cut hard: on chr22 the strict mask covers 25.2 Mb of 50.8, the Vindija FilterBed 19.6 Mb, and segmental duplications take out 4.3 Mb. Without them the step called sequence from positions the archaic genomes cannot be read at, and HG002's Neanderthal total was higher still: the filters took it from 115.2 Mb to 107.5 Mb, which is still 2.1 times the published median (see the same-person section under Runtime). The filters are part of the data version: a change to a pin or to a mask means `snpad2016-1kgp3-v3` and a rebuild. `setup.sh --archaic` also rebuilds a table whose format changed (its stamp holds the table format id of `og archaic`, `ARCHAIC_TABLE_FORMAT` in `setup.sh`), so a table built before the masked codes, or before the strict-mask rows were kept, is rebuilt on the next `setup.sh --archaic`. A table the last Python build code wrote has the same bytes, so its stamp is taken as current once and rewritten, and no install downloads the archaic genomes again for the move to Rust. Until then step 43 reads it as before: a table with no masked code gives empty masks. The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf`. A site that does not lift, or lifts to another chromosome, is dropped. The archaic files are downloaded one chromosome at a time (150 GB in all, at most 13.8 GB at once, for chromosome 2) and deleted once the chromosome is built, so an interrupted install resumes at the chromosome it stopped in. `ARCHAIC_REGIONS` (a GRCh37 BED, chromosomes named 1 to 22) builds only those regions. diff --git a/docs/44-ancient.md b/docs/44-ancient.md index 785913ca9..ec0c31253 100644 --- a/docs/44-ancient.md +++ b/docs/44-ancient.md @@ -14,7 +14,7 @@ Most people who look at their ancestry want to know where they fall among ancien - **smartpca** and **convertf** (EIGENSOFT 9.0.0): the PCA, and the conversion of the present-day people to EIGENSTRAT text. 9.0.0 is the first release that reads the AADR's TGENO genotype format. - **bcftools** (`convert --gvcf2vcf`, `view`, `query`) reads your genotypes at the panel's sites from the gVCF. -- **`bin/ancient_report.py`** (Python standard library): chooses the panel's people, lifts the AADR sites to GRCh38, adds your column to the present-day genotypes, aligns the axes and writes the report and the plot. +- **`og ancient`** (OwnGenome's Rust binary, `docs/rust.md`): chooses the panel's people, lifts the AADR sites to GRCh38, adds your column to the present-day genotypes, aligns the axes and writes the report and the plot. ## Docker Image diff --git a/docs/rust.md b/docs/rust.md index 241f92311..5c60ea02c 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -2,7 +2,7 @@ This page is for contributors. OwnGenome's own tools are moving from Python to Rust, one tool per pull request, into a single binary, `og`. This page says how the Rust code is laid out, how the binary reaches the pipeline, how it stays byte-for-byte compatible with the Python it replaces, and what CI checks. -Steps 04, 05, 07 and 25 call `og` (the filters of `og sv` and `og pgx`, and `og prs format`), step 06 and its Nextflow module `CLINVAR_SCREEN` call `og clinvar`, step 33 calls it (`og qc sex-loss`, the mosaic loss of Y or X), steps 23 and 31 and their Nextflow modules call it (`og vcf csq-rank` and `og table constraint-join`), step 27 and `CPIC_LOOKUP` call it (`og pgx cpic-report`), and so do steps 09b, 21 and 32: the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) replaced their Python. Steps 06, 17 and 31 and their Nextflow modules run the reads at the variant in `og`: the cDNA read check and scan (`og cdna check`, `og cdna scan`) and the phase of the compound-het pairs (`og comphet phase`); step 32 and its module read the BAM's contig names with `og bam contigs`. `scripts/setup.sh` calls it for the STRchive catalog, Exomiser's zips, the genetic map and the masked reference. Step 26b calls `og ancestry-fine`. Step 40 calls `og consumer`. Step 38 and the Nextflow family workflow call `og family`. So a run that reaches those steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull. The pieces below are the base every port builds on. +Steps 04, 05, 07 and 25 call `og` (the filters of `og sv` and `og pgx`, and `og prs format`), step 06 and its Nextflow module `CLINVAR_SCREEN` call `og clinvar`, step 33 calls it (`og qc sex-loss`, the mosaic loss of Y or X), steps 23 and 31 and their Nextflow modules call it (`og vcf csq-rank` and `og table constraint-join`), step 27 and `CPIC_LOOKUP` call it (`og pgx cpic-report`), and so do steps 09b, 21 and 32: the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) replaced their Python. Steps 06, 17 and 31 and their Nextflow modules run the reads at the variant in `og`: the cDNA read check and scan (`og cdna check`, `og cdna scan`) and the phase of the compound-het pairs (`og comphet phase`); step 32 and its module read the BAM's contig names with `og bam contigs`. `scripts/setup.sh` calls it for the STRchive catalog, Exomiser's zips, the genetic map and the masked reference. Step 26b calls `og ancestry-fine`, step 43 and `setup.sh --archaic` call `og archaic`, and step 44 and `setup.sh --ancient-panel` call `og ancient`. Step 40 calls `og consumer`. Step 38 and the Nextflow family workflow call `og family`. So a run that reaches those steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull. The pieces below are the base every port builds on. ## Layout @@ -26,7 +26,7 @@ rust/ crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read crates/og-bloodtype/ step 41, blood groups: og bloodtype report, check-markers, systems - crates/og-ancestry/ step 26b, fine-grained ancestry: og ancestry-fine + crates/og-ancestry/ steps 26b, 43 and 44: og ancestry-fine, og archaic, og ancient crates/og-reports/ the opt-in step reports of steps 45, 47, 48 and 50 to 53, and step 37's Y tree tools crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers @@ -36,7 +36,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. diff --git a/rust/README.md b/rust/README.md index 5071aad12..a6a412a2a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -18,7 +18,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-lookup` | the genotype lookup of `scripts/owngenome-lookup.sh`: `og lookup query`, `og lookup resolve`, `og lookup report` | | `crates/og-filters` | the VCF filters of steps 04, 05 and 07: `og sv svlen-fix`, `og sv inv-filter`, `og pgx refblock-fill`, `og pgx mito-merge` | | `crates/og-setup` | what `scripts/setup.sh` runs once: `og str catalog` (the STRchive catalog of steps 09 and 9b), `og setup unzip` (Exomiser's zips), `og setup genetic-map` (the genetic map of steps 26b and 51) | -| `crates/og-ancestry` | step 26b, fine-grained ancestry: `og ancestry-fine panel-map`, `check-map`, `order`, `sample-bed`, `chroms`, `report` | +| `crates/og-ancestry` | step 26b, fine-grained ancestry: `og ancestry-fine panel-map`, `check-map`, `order`, `sample-bed`, `chroms`, `report`; step 43's `og archaic table`, `strict`, `genotypes`, `report`; step 44's `og ancient select`, `lift`, `person-geno`, `align`, `report` | | `crates/og-steps` | step helpers, among them `og pgx outside-calls`, `og pgx hla-concordance` (steps 08, 36), `og pgx pypgx-summary` (step 32) and `og paraphase check-bam`, `og paraphase table` (step 35b) | | `crates/og-fresh` | `og dev freshness`: the upstream lookups of the monthly freshness issue, each answer typed, HTTP through the host's `curl` | | `crates/og-recheck` | `og recheck`: which of a finished sample's results a newer ClinVar or catalog would change, `--rerun`, `--affected`, `--launch-dir` | From 7cb54dc8ef4641e4d472ea7142a1c85cac92484f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:28:18 +0100 Subject: [PATCH 05/86] test(ancestry): a mask holds the 1-based bases of its BED intervals only Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 3d1b4e811..5e6a01b60 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -1609,6 +1609,20 @@ mod tests { } } + #[test] + fn a_mask_holds_its_bed_bases_only() { + // BED 99-100 and 200-300 are the 1-based bases 100 and 201 to 300. + let mut m = Mask { + iv: vec![(99, 100), (200, 300)], + i: 0, + }; + let got: Vec = [99, 100, 101, 200, 201, 300, 301] + .into_iter() + .filter(|&p| m.holds(p)) + .collect(); + assert_eq!(got, [100, 201, 300]); + } + #[test] fn gt_codes() { assert_eq!(gt_code("0/1"), Some(1)); From 3b492b9a30db6e823bae87486040d9c0462f94e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:28:18 +0100 Subject: [PATCH 06/86] test(ancestry): RED CONTROL, plant a 1-based mask and a chain block that holds its end Reverted in the next commit. Guard must go red: cargo test in og-ancestry and og-intervals, the archaic-table and ancient-lift parity cases, and tests/test_archaic_report.py. Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 2 +- rust/crates/og-intervals/src/chain.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 5e6a01b60..ff03f7791 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -224,7 +224,7 @@ struct Mask { impl Mask { fn holds(&mut self, pos: i64) -> bool { - let p = pos - 1; + let p = pos; while self.i < self.iv.len() && self.iv[self.i].1 <= p { self.i += 1; } diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs index 559a4d71e..160aaf502 100644 --- a/rust/crates/og-intervals/src/chain.rs +++ b/rust/crates/og-intervals/src/chain.rs @@ -240,10 +240,10 @@ impl Chain { // may still reach past p. let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); let mut first: Option<&Block> = None; - while j > 0 && i64::from(src.reach[j - 1]) > p { + while j > 0 && i64::from(src.reach[j - 1]) >= p { j -= 1; let b = &src.blocks[j]; - if i64::from(b.end) > p { + if i64::from(b.end) >= p { match first { None => first = Some(b), Some(f) if f.chain != b.chain => return None, From bb71d5692e77523235e27243e6453b9d4f3e4cf2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:38:15 +0100 Subject: [PATCH 07/86] test(ancestry): revert the red control of 3b492b9a Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 2 +- rust/crates/og-intervals/src/chain.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index ff03f7791..5e6a01b60 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -224,7 +224,7 @@ struct Mask { impl Mask { fn holds(&mut self, pos: i64) -> bool { - let p = pos; + let p = pos - 1; while self.i < self.iv.len() && self.iv[self.i].1 <= p { self.i += 1; } diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs index 160aaf502..559a4d71e 100644 --- a/rust/crates/og-intervals/src/chain.rs +++ b/rust/crates/og-intervals/src/chain.rs @@ -240,10 +240,10 @@ impl Chain { // may still reach past p. let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); let mut first: Option<&Block> = None; - while j > 0 && i64::from(src.reach[j - 1]) >= p { + while j > 0 && i64::from(src.reach[j - 1]) > p { j -= 1; let b = &src.blocks[j]; - if i64::from(b.end) >= p { + if i64::from(b.end) > p { match first { None => first = Some(b), Some(f) if f.chain != b.chain => return None, From 4ab9d0754bc8c11eff60d6230b4798544a6ac6d1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:41:17 +0100 Subject: [PATCH 08/86] fix(ancestry): join genotype calls in one position window --- .github/workflows/guard.yml | 18 ++ rust/crates/og-ancestry/src/archaic.rs | 263 ++++++++++++++++--------- tests/parity/cases.tsv | 2 +- tests/test_archaic_report.py | 31 +++ 4 files changed, 222 insertions(+), 92 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index e22eba988..0f2d5f70a 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -266,6 +266,24 @@ jobs: grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } echo "Control: the planted test failed, as it must." + - name: Control (archaic duplicate window must fail when equal positions are split) + run: | + source=crates/og-ancestry/src/archaic.rs + saved=$(mktemp "${RUNNER_TEMP}/archaic-window.XXXXXX") + cp "$source" "$saved" + trap 'cp -f "$saved" "$source"; rm -f "$saved"' EXIT + sed 's/r.pos == kept.pos/r.pos < kept.pos/' "$saved" > "$source" + grep -q 'r.pos < kept.pos' "$source" + log="${RUNNER_TEMP}/archaic-window-red.log" + if cargo test --locked --no-fail-fast -p og-ancestry duplicate_window_keeps_only_one_record_of_lookahead > "$log" 2>&1; then + cat "$log" + echo "ERROR: splitting a duplicate-position window passed" + exit 1 + fi + cat "$log" + grep -E 'duplicate_window_keeps_only_one_record_of_lookahead.*FAILED' "$log" + echo "Control: splitting the window failed its regression test." + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 5e6a01b60..4468fc815 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -25,7 +25,7 @@ //! Each command writes what `bin/archaic_report.py` wrote (checked by //! `tests/parity`). Deliberate changes: every output is written under a //! temporary name and renamed into place; a gzip input is told by its first -//! bytes, not its name; `strict` reads its rows in position order and stops +//! bytes, not its name; `strict` and `genotypes` read in position order and stop //! (exit 2) on a row that goes back, instead of keeping every position of a //! chromosome in memory; and malformed input is an `ERROR:` line, never a //! traceback. @@ -726,9 +726,17 @@ struct VcfRec { /// the reference. struct Vcf { text: Text, + previous: Option<(String, i64)>, } impl Vcf { + fn new(text: Text) -> Self { + Self { + text, + previous: None, + } + } + fn next(&mut self) -> Out> { loop { let Some(raw) = self.text.raw()? else { @@ -750,13 +758,28 @@ impl Vcf { } continue; } - if c.len() < 10 || c[3].chars().count() != 1 || c[4] == "." { + if c.len() < 10 { + continue; + } + let pos = need_int(c[1])?; + if let Some((chrom, prev)) = &self.previous + && chrom == c[0] + && pos < *prev + { + let message = format!("{}:{pos} comes after {prev}", c[0]); + return usage(format!( + "{} is not sorted: {message}", + self.text.path.display() + )); + } + self.previous = Some((c[0].to_owned(), pos)); + if c[3].chars().count() != 1 || c[4] == "." { continue; } if c[4].contains(',') || c[4].chars().count() != 1 { return Ok(Some(VcfRec { chrom: c[0].to_owned(), - pos: need_int(c[1])?, + pos, call: Sample { r#ref: c[3].to_owned(), alt: None, @@ -775,7 +798,7 @@ impl Vcf { } return Ok(Some(VcfRec { chrom: c[0].to_owned(), - pos: need_int(c[1])?, + pos, call: Sample { r#ref: c[3].to_owned(), alt: Some(c[4].to_owned()), @@ -786,39 +809,48 @@ impl Vcf { } } -/// The sample's calls on one chromosome, one per position after -/// [`pick_call`], sorted by position: a compact array, not a map. -fn settle(mut recs: Vec<(i64, Sample)>) -> Vec<(i64, Sample)> { - // A stable sort keeps the records of one position in file order. - recs.sort_by_key(|(p, _)| *p); - let mut out: Vec<(i64, Sample)> = Vec::new(); - let mut clashed = false; - for (pos, call) in recs { - match out.last_mut() { - Some((p, kept)) if *p == pos => { - if clashed { - continue; - } - let r#ref = call.r#ref.clone(); - match pick_call(Some(kept.clone()), call) { - Picked::Call(c) => *kept = c, - Picked::Clash => { - *kept = Sample { - r#ref, - alt: None, - code: 9, +/// Reduce one duplicate-position window in file order. Only the reduced +/// call and one record of lookahead survive, even for very long windows. +struct Calls { + vcf: Vcf, + pending: Option, +} + +impl Calls { + fn next(&mut self) -> Out> { + let first = match self.pending.take() { + Some(r) => Some(r), + None => self.vcf.next()?, + }; + let Some(mut kept) = first else { + return Ok(None); + }; + let mut clashed = false; + loop { + match self.vcf.next()? { + Some(r) if r.chrom == kept.chrom && r.pos == kept.pos => { + if !clashed { + let r#ref = r.call.r#ref.clone(); + kept.call = match pick_call(Some(kept.call), r.call) { + Picked::Call(c) => c, + Picked::Clash => { + clashed = true; + Sample { + r#ref, + alt: None, + code: 9, + } + } }; - clashed = true; } } - } - _ => { - clashed = false; - out.push((pos, call)); + other => { + self.pending = other; + return Ok(Some(kept)); + } } } } - out } /// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. @@ -893,13 +925,14 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; let vcf_path = a.req("--vcf"); - let mut vcf = Vcf { - text: Text::open(vcf_path)?, + let mut calls = Calls { + vcf: Vcf::new(Text::open(vcf_path)?), + pending: None, }; let mut spans: Vec = Vec::new(); let mut done: Vec = Vec::new(); - let mut pending = vcf.next()?; - while let Some(first) = pending.take() { + let mut pending = calls.next()?; + while let Some(first) = &pending { let chrom = first.chrom.clone(); if done.contains(&chrom) { return die(format!( @@ -908,33 +941,24 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } done.push(chrom.clone()); let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); - // The sample's calls on this chromosome; on a chromosome the table - // does not hold they are read past. - let mut recs = Vec::new(); - if table.is_some() { - recs.push((first.pos, first.call)); - } - loop { - match vcf.next()? { - Some(r) if r.chrom == chrom => { - if table.is_some() { - recs.push((r.pos, r.call)); - } - } - other => { - pending = other; - break; - } + if let Some(table) = table { + if let Some(span) = write_chrom( + &chrom, + table, + &mut calls, + &mut pending, + &head, + &pseudo, + af_col, + two_n, + out_dir, + )? { + spans.push(span); + } + } else { + while pending.as_ref().is_some_and(|r| r.chrom == chrom) { + pending = calls.next()?; } - } - let Some(table) = table else { - continue; - }; - let calls = settle(recs); - if let Some(span) = write_chrom( - &chrom, table, &calls, &head, &pseudo, af_col, two_n, out_dir, - )? { - spans.push(span); } } let mut text = @@ -967,7 +991,8 @@ type Span = (String, i64, i64, u64, u64, u64); fn write_chrom( chrom: &str, table: &str, - calls: &[(i64, Sample)], + calls: &mut Calls, + pending: &mut Option, head: &str, pseudo: &[String], af_col: usize, @@ -991,8 +1016,14 @@ fn write_chrom( let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); // Per archaic genome, the runs of written rows whose call is masked: // [first, last] positions, closed at the next unmasked row. - let mut runs: [Vec<(i64, i64)>; 3] = [Vec::new(), Vec::new(), Vec::new()]; + let mask_paths = + ARCHAICS.map(|name| Path::new(out_dir).join(format!("{chrom}.{name}.mask.bed"))); + let mut masks: Vec = mask_paths + .iter() + .map(|p| AtomicFile::create(p).map_err(|e| cannot_write(p, &e))) + .collect::>()?; let mut open_run: [Option<(i64, i64)>; 3] = [None; 3]; + let mut previous = None; let mut row = String::new(); while let Some(line) = t.next()? { let c: Vec<&str> = line.split('\t').collect(); @@ -1004,6 +1035,13 @@ fn write_chrom( )); } let pos = need_int(c[1])?; + if previous.is_some_and(|p| pos < p) { + return usage(format!( + "{table} is not sorted: {chrom}:{pos} comes after {}", + previous.unwrap() + )); + } + previous = Some(pos); let (r, alt) = (c[2], c[3]); let mut arch = [0u8; 3]; for k in 0..3 { @@ -1020,10 +1058,19 @@ fn write_chrom( } }; } - let code = match calls.binary_search_by_key(&pos, |(p, _)| *p) { - Err(_) => 0, - Ok(i) => { - let call = &calls[i].1; + while pending + .as_ref() + .is_some_and(|r| r.chrom == chrom && r.pos < pos) + { + *pending = calls.next()?; + } + let code = match pending + .as_ref() + .filter(|r| r.chrom == chrom && r.pos == pos) + { + None => 0, + Some(record) => { + let call = &record.call; if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { // Another allele than the row's: IBDmix's generate_gt // drops the line. @@ -1052,7 +1099,8 @@ fn write_chrom( Some(run) => run.1 = pos, } } else if let Some(run) = open_run[k].take() { - runs[k].push(run); + writeln!(masks[k], "{num}\t{}\t{}", run.0 - 1, run.1) + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } } row.clear(); @@ -1069,6 +1117,11 @@ fn write_chrom( last = pos; n += 1; } + // Read the tail too: an unsorted or unreadable record past the last + // table site must fail before committing this chromosome's outputs. + while pending.as_ref().is_some_and(|r| r.chrom == chrom) { + *pending = calls.next()?; + } if n == 0 { drop(gz); match std::fs::remove_file(&path) { @@ -1082,18 +1135,13 @@ fn write_chrom( gz.finish().and_then(AtomicFile::commit).map_err(werr)?; // IBDmix's mask (-r): the chromosome as in the genotype file, and a // line covers the positions start + 1 to end, as in a BED. - for k in 0..3 { - if let Some(run) = open_run[k] { - runs[k].push(run); + for (k, mut mask) in masks.into_iter().enumerate() { + if let Some((s, e)) = open_run[k] { + writeln!(mask, "{num}\t{}\t{e}", s - 1) + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } - let text: String = runs[k] - .iter() - .map(|(s, e)| format!("{num}\t{}\t{e}\n", s - 1)) - .collect(); - put( - &Path::new(out_dir).join(format!("{chrom}.{}.mask.bed", ARCHAICS[k])), - text.as_bytes(), - )?; + mask.commit() + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) } @@ -1155,7 +1203,7 @@ type Merged = (String, i64, i64, f64, Vec<&'static str>); /// Overlapping or touching segments of one class as one. fn merge(mut segs: Vec<(Seg, &'static str)>) -> Vec { - segs.sort_by(|(a, _), (b, _)| (chrom_key(&a.0), a.1, a.2).cmp(&(chrom_key(&b.0), b.1, b.2))); + segs.sort_by_key(|(a, _)| (chrom_key(&a.0), a.1, a.2)); let mut out: Vec = Vec::new(); for ((chrom, start, end, lod), who) in segs { match out.last_mut() { @@ -1205,7 +1253,7 @@ fn classify(per: &[Vec; 3]) -> Vec { rows.push((c, s, e, "Denisovan", vec!["Denisova"], l)); } } - rows.sort_by(|a, b| (chrom_key(&a.0), a.1).cmp(&(chrom_key(&b.0), b.1))); + rows.sort_by_key(|a| (chrom_key(&a.0), a.1)); rows } @@ -1659,16 +1707,49 @@ mod tests { #[test] fn a_clash_ends_its_position() { - let recs = vec![ - (4000, s("T", Some("C"), 1)), - (4000, s("T", Some("G"), 1)), - (4000, s("T", Some("C"), 1)), - (10, s("A", Some("G"), 2)), - ]; - assert_eq!( - settle(recs), - vec![(10, s("A", Some("G"), 2)), (4000, s("T", None, 9))] - ); + let mut calls = sample_calls(&[ + (10, "A", "G", "1/1"), + (4000, "T", "C", "0/1"), + (4000, "T", "G", "0/1"), + (4000, "T", "C", "0/1"), + ]); + assert_eq!(calls.next().unwrap().unwrap().call, s("A", Some("G"), 2)); + assert_eq!(calls.next().unwrap().unwrap().call, s("T", None, 9)); + assert!(calls.next().unwrap().is_none()); + } + + fn sample_calls(rows: &[(i64, &str, &str, &str)]) -> Calls { + let mut text = String::new(); + for (p, r, a, gt) in rows { + text += &format!("chr22\t{p}\t.\t{r}\t{a}\t.\tPASS\t.\tGT\t{gt}\n"); + } + Calls { + vcf: Vcf::new(Text { + path: "sample.vcf".into(), + lines: Lines::new(Box::new(io::Cursor::new(text.into_bytes())), Utf8::Strict), + line: String::new(), + }), + pending: None, + } + } + + #[test] + fn duplicate_window_keeps_only_one_record_of_lookahead() { + let mut rows = vec![(100, "A", "AT", "0/1"); 10_000]; + rows.push((100, "A", "G", "1/1")); + rows.push((200, "C", "T", "0/1")); + rows.push((150, "G", "A", "0/1")); // must not be read in the first window + let mut calls = sample_calls(&rows); + let first = calls.next().unwrap().unwrap(); + assert_eq!(first.pos, 100); + assert_eq!(first.call, s("A", Some("G"), 2)); + assert_eq!(calls.pending.as_ref().unwrap().pos, 200); + let error = calls + .next() + .err() + .expect("the second window sees the unsorted tail"); + assert_eq!(error.0, 2); + assert!(error.1.contains("not sorted")); } #[test] diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index dd63836b6..aae29404c 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -963,7 +963,7 @@ archaic-table empty-exclude gen:archaic-inputs.sh --chrom 22 --modern modern.tsv archaic-strict plain gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows.tsv --out s.tsv s.tsv bytes PYTHON_IMAGE archaic-strict unsorted gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows-unsorted.tsv --out s.tsv - bytes PYTHON_IMAGE archaic-strict empty-mask gen:archaic-inputs.sh --chrom 22 --strict empty.bed --rows rows.tsv --out s.tsv - bytes PYTHON_IMAGE -archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr22.Altai.mask.bed *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr21.Altai.mask.bed,gt/chr21.Vindija.mask.bed,gt/chr21.Denisova.mask.bed,gt/chr22.Altai.mask.bed,gt/chr22.Vindija.mask.bed,gt/chr22.Denisova.mask.bed *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes default gen:archaic-inputs.sh --table-dir . --vcf s.vcf.gz --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes afr-one gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop AFR --pseudo 1 --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes two-samples gen:archaic-inputs.sh --table-dir . --vcf two.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index 844658cd4..cffe87e77 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -315,6 +315,37 @@ def table_info(out, extra=()): r = run("genotypes", "--table-dir", bdir, "--vcf", f"{TMP}/m.vcf", "--out-dir", f"{TMP}/gtb") check("a code the table never holds (7) stops the step", r.returncode == 1 and "code '7'" in r.stderr, r.stderr) +# A streaming join must refuse backwards positions, including records past +# the table's end and records whose genotype it would otherwise ignore. +vcf_header = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" +for label, records in ( + ("within-table", [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")]), + ("past-table", [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")]), + ("ignored-call", [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")])): + bad_vcf = f"{TMP}/{label}.vcf" + write(bad_vcf, vcf_header + "".join( + f"chr22\t{p}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{fmt}\t0/1\n" + for p, ref, alt, fmt in records)) + out = f"{TMP}/order-{label}" + os.makedirs(out) + protected = ["chr22.gt.gz", *(f"chr22.{g}.mask.bed" for g in ARCHAICS)] + for name in protected: + write(f"{out}/{name}", "previous output\n") + r = run("genotypes", "--table-dir", tdir, "--vcf", bad_vcf, "--out-dir", out) + check(f"unsorted VCF {label} exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) + check(f"unsorted VCF {label} keeps prior genotype and masks", + all(open(f"{out}/{name}", "rb").read() == b"previous output\n" for name in protected)) + +unordered_table = f"{TMP}/table_unordered" +os.makedirs(unordered_table) +with gzip.open(f"{unordered_table}/chr22.tsv.gz", "wt") as f: + f.write(hdr + "\n".join([rows[1], rows[0], *rows[2:]]) + "\n") +r = run("genotypes", "--table-dir", unordered_table, "--vcf", f"{TMP}/s.vcf", + "--out-dir", f"{TMP}/gt_unordered") +check("an unsorted table exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) +check("an unsorted table leaves no partial genotype or mask", + not any(name.endswith((".gt.gz", ".mask.bed")) for name in os.listdir(f"{TMP}/gt_unordered"))) + # --- 3. report --------------------------------------------------------------------- idir = f"{TMP}/ibd" os.makedirs(idir) From b101631c6db40d1fe2318d33815b72a70dc0f33c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:44:17 +0100 Subject: [PATCH 09/86] docs(ancestry): record sorted-input contract and bounded memory --- CHANGELOG.md | 3 ++- docs/43-archaic.md | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08831d0a4..bfa29e96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 43's genotype files for 3 million table rows take 6 s and 159 MB instead of 12 s and 213 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. + - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 3e706e1ba..d9689a32c 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -72,6 +72,12 @@ The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf` - `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A site with no record is read as the reference allele, a record whose FILTER is not PASS as unknown, and a site where you carry another ALT than the table's is left out. - The GENCODE gene lines `setup.sh` installs, for the genes of each segment (without them the segments are listed without genes) +`og archaic genotypes` reads the VCF and table together, keeping one +duplicate-position window. Positions within each chromosome must be in +ascending order in both inputs. A position that goes backwards stops the +command with exit 2. Mask runs are written as they close. Duplicate calls, +clashes, masked sites and the output columns keep their previous meaning. + ## Command ```bash From c47cd892115bdc1c0c55897d72bce3845b86004d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:17:11 +0100 Subject: [PATCH 10/86] feat(summary): whole summary.json and its post pass in og summary collect Without --only, og summary collect reads all three groups over one walk of the folder and runs the post pass: the readings that join sections, the cyp2d6 block, not_run, not_assessed, the database re-check and new_since. A part of the post pass that fails makes only its own section unreadable; the summary is still written. Bead: pgp-bs9u.44 --- rust/Cargo.lock | 1 + rust/crates/og-summary/Cargo.toml | 1 + rust/crates/og-summary/src/collect.rs | 257 ++++++++----- rust/crates/og-summary/src/collect/group_b.rs | 108 +++--- rust/crates/og-summary/src/collect/group_c.rs | 46 ++- .../og-summary/src/collect/group_c/repeats.rs | 15 +- rust/crates/og-summary/src/collect/model.rs | 14 + rust/crates/og-summary/src/collect/post.rs | 339 +++++++++++++++++ .../og-summary/src/collect/post/tests.rs | 355 ++++++++++++++++++ rust/crates/og-summary/src/collect/tests.rs | 12 +- 10 files changed, 974 insertions(+), 174 deletions(-) create mode 100644 rust/crates/og-summary/src/collect/post.rs create mode 100644 rust/crates/og-summary/src/collect/post/tests.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c5d9ae54a..a99006d13 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -846,6 +846,7 @@ dependencies = [ "og-clinvar", "og-cn", "og-io", + "og-runlog", "og-steps", "pycompat", ] diff --git a/rust/crates/og-summary/Cargo.toml b/rust/crates/og-summary/Cargo.toml index 325b1f475..dedbfb695 100644 --- a/rust/crates/og-summary/Cargo.toml +++ b/rust/crates/og-summary/Cargo.toml @@ -15,6 +15,7 @@ og-steps.workspace = true og-ancestry.workspace = true og-chip.workspace = true og-cn.workspace = true +og-runlog.workspace = true indexmap.workspace = true memchr.workspace = true diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index e351fd308..13b22eb76 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -14,6 +14,10 @@ //! ancestry, structural variants, repeats, telomeres, runs of //! homozygosity, the haplogroups, the opt-in step tables, chrM, traits with //! the APOL1 G2 deletion, and the couple screen ([`group_c`]). +//! Without `--only` it reads all three groups and runs the post pass +//! ([`post`]): the readings that join sections, the steps not run, what no +//! step assessed, the last database re-check and what changed since the +//! previous run. That is the whole summary.json. //! //! How it reads a folder: one walk of the folder ([`scan::FolderIndex`]) //! instead of a `glob` per candidate path; each input read once (the @@ -32,6 +36,7 @@ pub mod group_b; pub mod group_c; pub mod model; pub mod pgx; +pub mod post; pub mod scan; pub mod sections; pub mod text; @@ -47,17 +52,17 @@ use std::io::Write; use std::path::Path; const USAGE: &str = - "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only {a,b,c} - --out OUT [--declared-sex DECLARED_SEX] [--assets ASSETS] + "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] + [--declared-sex DECLARED_SEX] [--assets ASSETS] [--only {a,b,c}] "; const OPTS: &[Opt] = &[ Opt("--sample", Kind::Value, true), Opt("--sample-dir", Kind::Value, true), Opt("--declared-sex", Kind::Value, false), - Opt("--out", Kind::Value, true), + Opt("--out", Kind::Value, false), Opt("--assets", Kind::Value, false), - Opt("--only", Kind::Choice(&["a", "b", "c"]), true), + Opt("--only", Kind::Choice(&["a", "b", "c"]), false), ]; /// What one collect reads. @@ -70,7 +75,7 @@ pub struct Input { pub declared_sex: Option, /// The repository's `assets/` folder. pub assets: String, - /// The group of sections to read: `a`, `b` or `c`. + /// The group of sections to read: `a`, `b`, `c`, or all of them. pub group: Group, /// The folder of the two tables group c reads from `bin/`. pub tables: String, @@ -82,6 +87,8 @@ pub enum Group { A, B, C, + /// Every section and the post pass: the whole summary.json. + All, } /// `og summary collect`: exit 0 when summary.json is written (whatever @@ -102,28 +109,39 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { declared_sex: a.get("--declared-sex").map(str::to_owned), assets: a.get("--assets").map_or_else(default_assets, str::to_owned), tables: group_c::util::default_tables(), - group: match a.text("--only") { - "b" => Group::B, - "c" => Group::C, - _ => Group::A, + group: match a.get("--only") { + Some("a") => Group::A, + Some("b") => Group::B, + Some("c") => Group::C, + _ => Group::All, }, }; // A part of the sections must never replace the sample's complete - // summary.json, so there is no default path while `--only` exists. - let target = a.text("--out").to_owned(); - let read = match a.text("--only") { - "b" => group_b::collect, - _ => collect, - }; - let summary = match read(&input) { - Ok(s) => s, - Err(e) => return crate::finish(Err(e), err), + // summary.json, so a part is written only where `--out` says. + let target = match a.get("--out") { + Some(t) => t.to_owned(), + None if input.group == Group::All => crate::join(&input.sample_dir, "summary.json"), + None => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og summary collect: error: --only needs --out: a part of the sections never replaces summary.json" + ); + return 2; + } }; // The sections' own data is freed before the text is made, and the // tree before the file is written: a large ClinVar list is in memory in // one form at a time. - let value = summary.to_json(); - drop(summary); + let read = match input.group { + Group::All => post::collect(&input), + Group::B => group_b::collect(&input).map(|s| s.to_json()), + Group::A | Group::C => collect(&input).map(|s| s.to_json()), + }; + let value = match read { + Ok(v) => v, + Err(e) => return crate::finish(Err(e), err), + }; let body = match dumps( &value, &Dump { @@ -161,47 +179,83 @@ fn default_assets() -> String { .to_string() } -/// Reads the folder into the summary of group a. An error is a run status -/// or a manifest that is there and cannot be read: without it a result of -/// an earlier run would be called current, so no summary is written. +/// What every group reads first: one walk of the folder, the run status +/// and the manifest. +pub struct Folder { + pub index: FolderIndex, + pub run: Option, + pub manifest: Option, +} + +impl Folder { + /// Walks the folder and reads its run status and manifest. An error is + /// a run status or a manifest that is there and cannot be read: without + /// it a result of an earlier run would be called current, so no summary + /// is written. + pub fn open(input: &Input) -> Result { + let dir = Path::new(&input.sample_dir); + let index = FolderIndex::walk(dir); + let run = + read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); + let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + Ok(Folder { + index, + run, + manifest, + }) + } +} + +/// Reads the folder into the summary of group a or group c. pub fn collect(input: &Input) -> Result { - let dir = Path::new(&input.sample_dir); - let index = FolderIndex::walk(dir); - let run = - read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); - let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); - let s = pycompat::mtime::escape(&input.sample); - let find = |pats: &[String]| index.first(pats).map(str::to_owned); - let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; - if input.group == Group::C { + let folder = Folder::open(input)?; + let (sections, declared_sex) = if input.group == Group::C { // The declared sex the chrY filter of CNVpytor reads comes from // step 33's table when the run status and manifest have none. - let qc = find(&[format!("qc/{s}_sample_qc.tsv")]).map(file); - let qc_declared = qc.and_then(|(p, _)| { + let s = pycompat::mtime::escape(&input.sample); + let qc = folder + .index + .first(&[format!("qc/{s}_sample_qc.tsv")]) + .map(|rel| crate::join(&input.sample_dir, rel)); + let qc_declared = qc.and_then(|p| { let rows = sections::read_tsv(&p).ok()?; SampleQc::read(&p, &rows) .ok()? .declared_sex() .map(str::to_owned) }); - let declared_sex = declared( + let declared_sex = declared(input, &folder, qc_declared.as_deref()); + let mut sections = read_c(input, &folder); + post_c( input, - run.as_ref(), - manifest.as_ref(), - qc_declared.as_deref(), + &folder.index, + &mut sections, + &group_c::Others::default(), + declared_sex.as_deref(), ); - let sections = read_group_c(input, &index, run.as_ref(), declared_sex.as_deref()); - return Ok(Summary { - sample: input.sample.clone(), - generated_utc: time::utc_minute(now()).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6: None, - }); - } + (sections, declared_sex) + } else { + let sections = read_a(input, &folder); + let declared_sex = declared(input, &folder, qc_declared(§ions)); + (sections, declared_sex) + }; + Ok(Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now()).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6: None, + }) +} +/// Reads the sections of group a, in [`GROUP_A`] order: the sample VCF on +/// its own thread while the other six are read. +pub fn read_a(input: &Input, folder: &Folder) -> Vec
{ + let s = pycompat::mtime::escape(&input.sample); + let find = |pats: &[String]| folder.index.first(pats).map(str::to_owned); + let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; let variants_at = find(&[format!("vcf/{s}.vcf.gz")]); let (variants, rest) = std::thread::scope(|scope| { let v = scope.spawn(|| { @@ -233,29 +287,23 @@ pub fn collect(input: &Input) -> Result { read_count, variants, ]; - let sections: Vec
= GROUP_A + GROUP_A .iter() .zip(reads) - .map(|(spec, r): (&Spec, _)| Section::new(*spec, r, run.as_ref())) - .collect(); + .map(|(spec, r): (&Spec, _)| Section::new(*spec, r, folder.run.as_ref())) + .collect() +} - let qc_declared = sections +/// The declared sex step 33's table records, from its section (read when +/// the section is ok or stale). +pub fn qc_declared(sections: &[Section]) -> Option<&str> { + sections .iter() .find(|s| s.spec.key == "sample_qc") .and_then(|s| match &s.data { Some(Data::SampleQc(q)) => q.declared_sex(), _ => None, - }); - let declared_sex = declared(input, run.as_ref(), manifest.as_ref(), qc_declared); - Ok(Summary { - sample: input.sample.clone(), - generated_utc: time::utc_minute(now()).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6: None, - }) + }) } fn now() -> f64 { @@ -266,16 +314,14 @@ fn now() -> f64 { /// The declared sex: the command line's, the run status's, the /// manifest's, then step 33's; the first that is not empty. -fn declared( - input: &Input, - run: Option<&RunStatus>, - manifest: Option<&Manifest>, - qc: Option<&str>, -) -> Option { +pub fn declared(input: &Input, folder: &Folder, qc: Option<&str>) -> Option { [ input.declared_sex.as_deref(), - run.and_then(|r| r.declared_sex.as_deref()), - manifest.and_then(|m| m.value("run", "declared_sex")), + folder.run.as_ref().and_then(|r| r.declared_sex.as_deref()), + folder + .manifest + .as_ref() + .and_then(|m| m.value("run", "declared_sex")), qc, ] .into_iter() @@ -284,39 +330,56 @@ fn declared( .map(str::to_owned) } -/// Reads the sections of group c, then runs the parts of the post pass -/// that change them. -fn read_group_c( +/// Reads the sections of group c, in [`group_c::GROUP_C`] order. +pub fn read_c(input: &Input, folder: &Folder) -> Vec
{ + let c = ctx_c(input, &folder.index); + group_c::read_all(&c) + .into_iter() + .zip(group_c::GROUP_C.iter()) + .map(|(r, spec)| { + let r = r.and_then(|found| match found { + None => Ok(None), + Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), + }); + Section::new(*spec, r, folder.run.as_ref()) + }) + .collect() +} + +/// The parts of the post pass that change group c's sections: the repeat +/// readings, Locityper's primary calls and CNVpytor's chrY filter. A part +/// that fails makes its own section unreadable, with a note that names the +/// file and the reading; the other sections and the summary stay. +pub fn post_c( input: &Input, index: &FolderIndex, - run: Option<&RunStatus>, + sections: &mut [Section], + others: &group_c::Others, declared_sex: Option<&str>, -) -> Vec
{ - let c = group_c::Ctx { +) { + let c = ctx_c(input, index); + let failed = { + let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections + .iter_mut() + .map(|s| (s.spec, s.state, s.data.as_mut())) + .collect(); + group_c::post(&c, &mut view, others, declared_sex) + }; + for (key, why) in failed { + if let Some(s) = sections.iter_mut().find(|s| s.spec.key == key) { + s.make_unreadable(&why); + } + } +} + +fn ctx_c<'a>(input: &'a Input, index: &'a FolderIndex) -> group_c::Ctx<'a> { + group_c::Ctx { sample: &input.sample, dir: &input.sample_dir, assets: &input.assets, tables: &input.tables, index, - }; - let reads = group_c::read_all(&c); - let mut sections: Vec
= group_c::GROUP_C - .iter() - .zip(reads) - .map(|(spec, r)| { - let r = r.and_then(|found| match found { - None => Ok(None), - Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), - }); - Section::new(*spec, r, run) - }) - .collect(); - let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections - .iter_mut() - .map(|s| (s.spec, s.state, s.data.as_mut())) - .collect(); - group_c::post(&c, &mut view, &group_c::Others::default(), declared_sex); - sections + } } /// What one reader gave: the file (relative), its mtime and its data. diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs index cb0ba8a1e..394178cf5 100644 --- a/rust/crates/og-summary/src/collect/group_b.rs +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -14,13 +14,12 @@ use super::carrier::Carrier; use super::clinical::{Clinical, Slivar}; use super::clinvar::{ClinVar, Floor, HitsVcf}; use super::cpsr::{self, Cpsr}; -use super::model::{Manifest, RunStatus, Section, Spec, Summary}; +use super::model::{Section, Spec, Summary}; use super::pgx::{Cpic, Cyp2d6, Cyrius, PharmCat, Pypgx}; use super::scan::FolderIndex; use super::sections::{Data, SampleQc}; use super::two_hits::{self, DeletionInputs}; -use super::{Input, Read, read_at, read_optional, with_mtime}; -use std::path::Path; +use super::{Folder, Input, Read, read_at, with_mtime}; /// The sections `og summary collect --only b` reads, in summary.json's order. pub const GROUP_B: [Spec; 9] = [ @@ -74,12 +73,42 @@ pub const GROUP_B: [Spec; 9] = [ /// Reads the folder into the summary of group b. An error is a run status /// or a manifest that is there and cannot be read. pub fn collect(input: &Input) -> Result { - let dir = Path::new(&input.sample_dir); + let folder = Folder::open(input)?; + let sections = read(input, &folder); + let cyp2d6 = cyp2d6(input, &folder.index, §ions); + let s = pycompat::mtime::escape(&input.sample); + // The run block is group a's: the declared sex from the command line, + // the run status, the manifest or step 33's table, the first that has one. + let qc_declared = folder + .index + .first(&[format!("qc/{s}_sample_qc.tsv")]) + .and_then(|rel| { + let p = crate::join(&input.sample_dir, rel); + let rows = super::sections::read_tsv(&p).ok()?; + SampleQc::read(&p, &rows) + .ok()? + .declared_sex() + .map(str::to_owned) + }); + let declared_sex = super::declared(input, &folder, qc_declared.as_deref()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + Ok(Summary { + sample: input.sample.clone(), + generated_utc: super::time::utc_minute(now).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6, + }) +} + +/// Reads the sections of group b, in [`GROUP_B`] order. +pub fn read(input: &Input, folder: &Folder) -> Vec
{ let d = input.sample_dir.as_str(); - let index = FolderIndex::walk(dir); - let run = - read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); - let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + let index = &folder.index; let s = pycompat::mtime::escape(&input.sample); let raw = input.sample.as_str(); let file = |rel: &str| -> (String, String) { (crate::join(d, rel), rel.to_owned()) }; @@ -122,7 +151,7 @@ pub fn collect(input: &Input) -> Result { )?; with_mtime(&p, rel, Data::Clinical(data)) }); - let rest = read_rest(input, &s, &index, &find, &hits, &deletions); + let rest = read_rest(input, &s, index, &find, &hits, &deletions); let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { h.join() .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) @@ -133,55 +162,26 @@ pub fn collect(input: &Input) -> Result { let reads = [ clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, ]; - let sections: Vec
= GROUP_B + GROUP_B .iter() .zip(reads) - .map(|(spec, r)| Section::new(*spec, r, run.as_ref())) - .collect(); + .map(|(spec, r)| Section::new(*spec, r, folder.run.as_ref())) + .collect() +} + +/// The `cyp2d6` block: CPIC, pypgx and Cyrius side by side, with step 36's +/// row. `None` when `sections` lack one of the three. +pub fn cyp2d6(input: &Input, index: &FolderIndex, sections: &[Section]) -> Option { + let s = pycompat::mtime::escape(&input.sample); + let d = input.sample_dir.as_str(); let sec = |k: &str| sections.iter().find(|x| x.spec.key == k); - let cyp2d6 = match (sec("cpic"), sec("pypgx"), sec("cyrius")) { - (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new( - c, - p, - y, - one(format!("pgx_consensus/{s}_pgx_consensus.tsv")), - d, - )), + let consensus = index + .first(&[format!("pgx_consensus/{s}_pgx_consensus.tsv")]) + .map(|rel| (crate::join(d, rel), rel.to_owned())); + match (sec("cpic"), sec("pypgx"), sec("cyrius")) { + (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new(c, p, y, consensus, d)), _ => None, - }; - - // The run block is group a's: the declared sex from the command line, - // the run status, the manifest or step 33's table, the first that has one. - let qc_declared = one(format!("qc/{s}_sample_qc.tsv")).and_then(|(p, _)| { - let rows = super::sections::read_tsv(&p).ok()?; - SampleQc::read(&p, &rows) - .ok()? - .declared_sex() - .map(str::to_owned) - }); - let declared_sex = [ - input.declared_sex.clone(), - run.as_ref().and_then(|r| r.declared_sex.clone()), - manifest - .as_ref() - .and_then(|m| m.value("run", "declared_sex").map(str::to_owned)), - qc_declared, - ] - .into_iter() - .flatten() - .find(|v| !v.is_empty()); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0.0, |d| d.as_secs_f64()); - Ok(Summary { - sample: input.sample.clone(), - generated_utc: super::time::utc_minute(now).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6, - }) + } } /// Finds a section's file: the full path and the path in the folder. diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 15ab27782..3450f6b5d 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -9,7 +9,8 @@ //! readings with the traits and CPIC states, Locityper's primary calls, and //! the chrY filter of CNVpytor's count for a declared female. What they //! need from group b (the CPIC state, pypgx and Cyrius) comes in -//! [`Others`]; `--only c` reads none of it, so it is empty there. +//! [`Others`]; `--only c` reads none of it, so it is empty there. A part +//! that fails makes only its own section unreadable. mod chrm; mod cn; @@ -236,13 +237,15 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { /// The post pass over the read sections (`sections` in [`GROUP_C`] /// order, each with its state and data): the repeat readings with the /// traits and CPIC states, each Locityper locus's primary call, and the -/// chrY filter of CNVpytor's count. +/// chrY filter of CNVpytor's count. Each part runs on its own: one that +/// fails (an error or a panic) is returned as its section's key and why, +/// and the others still run. pub fn post( c: &Ctx<'_>, sections: &mut [(Spec, State, Option<&mut Data>)], others: &Others, declared_sex: Option<&str>, -) { +) -> Vec<(&'static str, String)> { let state = |key: &str, secs: &[(Spec, State, Option<&mut Data>)]| { secs.iter() .find(|(s, ..)| s.key == key) @@ -257,19 +260,42 @@ pub fn post( .and_then(|x| x.2.as_deref()) .and_then(json) .cloned(); + let mut failed = Vec::new(); for (spec, st, data) in sections.iter_mut() { let live = matches!(st, State::Ok | State::Stale); let Some(Data::Json(v)) = data.as_deref_mut() else { continue; }; - match spec.key { - "expansions" if live => repeats::post(c, v, traits_state, cpic_state), - "locityper" if live => { - tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + let part = || -> Result<(), String> { + match spec.key { + "expansions" if live => repeats::post(c, v, traits_state, cpic_state) + .map_err(|e| format!("the repeat readings could not be made: {e}")), + "locityper" if live => { + tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + Ok(()) + } + "cnvpytor" => { + sv::cnvpytor_sex_filter(v, declared_sex); + Ok(()) + } + _ => Ok(()), } - "cnvpytor" => sv::cnvpytor_sex_filter(v, declared_sex), - _ => {} - } + }; + failed.extend(contain(spec.key, part)); + } + failed +} + +/// Runs one part of the post pass: `None` when it went through, else its +/// section's key and why it failed, for an error and for a panic alike. +pub fn contain( + key: &'static str, + part: impl FnOnce() -> Result<(), String>, +) -> Option<(&'static str, String)> { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { + Ok(Ok(())) => None, + Ok(Err(e)) => Some((key, e)), + Err(_) => Some((key, format!("internal error: the post pass of {key} stopped"))), } } diff --git a/rust/crates/og-summary/src/collect/group_c/repeats.rs b/rust/crates/og-summary/src/collect/group_c/repeats.rs index 9d9610fd0..3081543d3 100644 --- a/rust/crates/og-summary/src/collect/group_c/repeats.rs +++ b/rust/crates/og-summary/src/collect/group_c/repeats.rs @@ -910,9 +910,10 @@ fn readings( } /// The post pass: the readings again, with the traits and CPIC states, over -/// the loci as the section lists them. -pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { - let Value::Object(m) = v else { return }; +/// the loci as the section lists them. An error is a reading that could +/// not be made; the section's data is then left as it was. +pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) -> Result<(), String> { + let Value::Object(m) = v else { return Ok(()) }; let loci: Vec<(String, Obj)> = match m.get("loci") { Some(Value::Array(xs)) => xs .iter() @@ -924,10 +925,10 @@ pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { _ => None, }) .collect(), - _ => return, + _ => return Ok(()), }; let by = by_locus(loci.iter().map(|(k, o)| (k, o))); - if let Ok(r) = readings(c, &by, traits, cpic) { - m.insert("readings".into(), r); - } + let r = readings(c, &by, traits, cpic)?; + m.insert("readings".into(), r); + Ok(()) } diff --git a/rust/crates/og-summary/src/collect/model.rs b/rust/crates/og-summary/src/collect/model.rs index c6d6e22a3..2d6e7c994 100644 --- a/rust/crates/og-summary/src/collect/model.rs +++ b/rust/crates/og-summary/src/collect/model.rs @@ -178,6 +178,20 @@ impl Section { s } + /// Marks a section whose reading failed after its file was read (a + /// part of the post pass): `unreadable`, as a reader's error leaves it, + /// with a note that names the file the section came from. + pub fn make_unreadable(&mut self, why: &str) { + self.note = Some(match self.source.as_deref() { + Some(src) => format!("{}: {why}", crate::basename(src)), + None => why.to_owned(), + }); + self.state = State::Unreadable; + self.source = None; + self.file_date = None; + self.data = None; + } + pub fn to_json(&self) -> Value { let mut m = IndexMap::new(); m.insert("title".into(), Value::Str(self.spec.title.into())); diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs new file mode 100644 index 000000000..42b9fe1ad --- /dev/null +++ b/rust/crates/og-summary/src/collect/post.rs @@ -0,0 +1,339 @@ +//! The whole summary.json: every section of groups a, b and c, read over +//! one walk of the folder, then the post pass. +//! +//! The post pass is what joins sections after each is read: +//! +//! - the repeat readings with the traits and CPIC states, Locityper's +//! primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter +//! with the declared sex ([`super::post_c`]); +//! - the `cyp2d6` block over CPIC, pypgx and Cyrius; +//! - `not_run`: the steps whose section has no file; +//! - `not_assessed`: what no step looked at, with the lines that depend on +//! the polygenic scores, HBA1/HBA2, mobile elements, VNtyper and MoChA; +//! - `recheck`: the last database re-check of the sample; +//! - `new_since`: what changed since the previous run kept in `history/`. +//! +//! Each part that changes a section runs on its own: when it fails, only +//! its section becomes `unreadable`, with a note naming the file, and the +//! summary is still written. The Python collector ran this pass outside +//! its per-section error handling, so one failure there lost summary.json +//! and both reports. A previous run that cannot be read leaves `new_since` +//! out, as before; the comparison itself has no error to hide. + +use super::group_c::Others; +use super::model::{Section, State, Summary}; +use super::sections::Data; +use super::{Folder, Input, declared, group_b, post_c, qc_declared, read_a, read_c, time}; +use og_runlog::{delta, files}; +use pycompat::json::Value; +use pycompat::value::truthy; +use std::path::Path; + +/// summary.json's sections, in its order. +pub const ORDER: [&str; 48] = [ + "coverage", + "callability", + "sex_check", + "sample_qc", + "mosaic_sex_loss", + "read_count", + "variants", + "clinvar", + "couple", + "carrier", + "pharmcat", + "cpic", + "pypgx", + "cyrius", + "parascopy", + "paralogs", + "hla", + "prs", + "ancestry_fine", + "manta", + "delly", + "cnvpytor", + "sv_consensus", + "annotsv", + "expansions", + "telomere", + "roh", + "haplogroup", + "y_haplogroup", + "y_tree", + "y_str", + "consumer_export", + "bloodtype", + "lpa", + "vntyper", + "locityper", + "mito", + "traits", + "archaic", + "ancient", + "viral", + "exomiser", + "chip", + "mei", + "mosaic", + "cpsr", + "clinical", + "slivar", +]; + +/// Sections that are no step of their own, so never in `not_run`: the read +/// count is written by alignment, which a run from a BAM or a VCF never +/// does; the couple screen and the carrier list come with other steps. +const NOT_A_STEP: [&str; 3] = ["read_count", "couple", "carrier"]; + +/// What no step assesses. +const NOT_ASSESSED: [&str; 5] = [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", +]; + +/// Added unless step 25 ran with an ancestry reference panel. +const PRS_NOT_ADJUSTED: &str = + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums"; + +/// Added unless step 35's depth count (short reads) or step 35b +/// (Paraphase, PacBio HiFi, both genes) made a call. +const HBA_NOT_ASSESSED: &str = "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count \ +(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call"; + +/// Added when step 50 (opt-in, MEGAnE) did not run. A failed step 50 is +/// listed under not run, and an unreadable table under its section. +const MEI_NOT_ASSESSED: &str = "Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): \ +not looked for (step 50, MEGAnE, is opt-in)"; + +/// Added unless step 52 (VNtyper) gave a positive, possible or negative +/// reading: no other step sees a frameshift inside the MUC1 VNTR. +const VNTYPER_NOT_ASSESSED: &str = "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; \ +no standard caller sees them"; + +/// Added unless step 51 (MoChA) assessed the sample. +const MCA_NOT_ASSESSED: &str = "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share \ +of the blood cells): step 51 (MoChA) did not assess this sample"; + +/// Reads the whole summary of a sample folder. The three groups are read +/// at once over one walk of the folder. An error is a run status or a +/// manifest that is there and cannot be read. +pub fn collect(input: &Input) -> Result { + let folder = Folder::open(input)?; + let (a, b, mut c) = std::thread::scope(|scope| { + let a = scope.spawn(|| read_a(input, &folder)); + let b = scope.spawn(|| group_b::read(input, &folder)); + let c = read_c(input, &folder); + let stopped = |specs: &[super::model::Spec]| -> Vec
{ + specs + .iter() + .map(|s| { + Section::new( + *s, + Err("internal error: the section reader stopped".into()), + folder.run.as_ref(), + ) + }) + .collect() + }; + let a = a + .join() + .unwrap_or_else(|_| stopped(&super::model::GROUP_A)); + let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); + (a, b, c) + }); + let declared_sex = declared(input, &folder, qc_declared(&a)); + post_c( + input, + &folder.index, + &mut c, + &others(&b), + declared_sex.as_deref(), + ); + let cyp2d6 = group_b::cyp2d6(input, &folder.index, &b); + let sections = in_order(a.into_iter().chain(b).chain(c).collect()); + let not_run = not_run(§ions); + let not_assessed = not_assessed(§ions); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + let summary = Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6, + }; + let mut v = summary.to_json(); + drop(summary); + let Value::Object(m) = &mut v else { + return Err("internal error: the summary is not an object".into()); + }; + m.insert( + "not_run".into(), + Value::Array(not_run.into_iter().map(Value::Str).collect()), + ); + m.insert( + "not_assessed".into(), + Value::Array( + not_assessed + .into_iter() + .map(|x| Value::Str(x.to_owned())) + .collect(), + ), + ); + let dir = Path::new(&input.sample_dir); + if let Some(r) = delta::read_recheck(&dir.join("recheck").join("state.json")) { + m.insert("recheck".into(), Value::Object(r)); + } + if let Some(n) = new_since(dir, &v) { + if let Value::Object(m) = &mut v { + m.insert("new_since".into(), n); + } + } + Ok(v) +} + +/// What the post pass of group c needs from group b: the CPIC state, +/// pypgx's CYP2D6 call and Cyrius's genotype and Filter. +fn others(b: &[Section]) -> Others { + let sec = |k: &str| b.iter().find(|s| s.spec.key == k); + let live = |s: &Section| matches!(s.state, State::Ok | State::Stale); + Others { + cpic: sec("cpic").map(|s| s.state), + traits: None, + pypgx: sec("pypgx") + .filter(|s| live(s)) + .and_then(|s| match &s.data { + Some(Data::Pypgx(p)) => Some((s.state, p.cyp2d6.clone())), + _ => None, + }), + cyrius: sec("cyrius") + .filter(|s| live(s)) + .and_then(|s| match &s.data { + Some(Data::Cyrius(c)) => Some((s.state, Some(c.genotype.clone()), c.filter.clone())), + _ => None, + }), + } +} + +/// The sections in summary.json's order ([`ORDER`]). +fn in_order(mut sections: Vec
) -> Vec
{ + sections.sort_by_key(|s| ORDER.iter().position(|k| *k == s.spec.key)); + sections +} + +/// The steps whose section has no file, with " (step N failed)" for a step +/// that failed in the last run. +fn not_run(sections: &[Section]) -> Vec { + sections + .iter() + .filter(|s| matches!(s.state, State::Missing | State::Failed)) + .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) + .map(|s| { + if s.state == State::Failed { + format!("{} (step {} failed)", s.spec.title, s.spec.step) + } else { + s.spec.title.to_owned() + } + }) + .collect() +} + +/// What no step assessed for this sample. +fn not_assessed(sections: &[Section]) -> Vec<&'static str> { + let sec = |k: &str| sections.iter().find(|s| s.spec.key == k); + let state = |k: &str| sec(k).map_or(State::Missing, |s| s.state); + // A section's data is there when it is ok or stale; else it reads {}. + let field = |k: &str, key: &str| -> Option<&Value> { + match sec(k).and_then(|s| s.data.as_ref()) { + Some(Data::Json(Value::Object(m))) => m.get(key), + _ => None, + } + }; + let mut out = NOT_ASSESSED.to_vec(); + if !field("prs", "adjusted").is_some_and(truthy) { + out.push(PRS_NOT_ADJUSTED); + } + // Step 35b on a region subset (--paraphase_regions) may leave hba out. + let hba_genes: Vec<&str> = match field("paralogs", "genes") { + Some(Value::Array(genes)) => genes + .iter() + .filter(|g| is(at(Some(g), "region"), "hba") && is(at(Some(g), "status"), "called")) + .filter_map(|g| match at(Some(g), "gene") { + Some(Value::Str(x)) => Some(x.as_str()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + let paraphase_hba = ["HBA1", "HBA2"].iter().all(|g| hba_genes.contains(g)); + if !(paraphase_hba || is(at(field("parascopy", "hba"), "status"), "called")) { + out.push(HBA_NOT_ASSESSED); + } + if state("mei") == State::Missing { + out.push(MEI_NOT_ASSESSED); + } + let vntyper_read = state("vntyper") == State::Ok + && ["positive", "possible", "negative"] + .iter() + .any(|w| is(field("vntyper", "status"), w)); + if !vntyper_read { + out.push(VNTYPER_NOT_ASSESSED); + } + if !is(field("mosaic", "status"), "assessed") { + out.push(MCA_NOT_ASSESSED); + } + out +} + +/// True when `v` is the string `word`. +fn is(v: Option<&Value>, word: &str) -> bool { + matches!(v, Some(Value::Str(x)) if x == word) +} + +/// `v[key]` when `v` is an object that has it. +fn at<'a>(v: Option<&'a Value>, key: &str) -> Option<&'a Value> { + match v { + Some(Value::Object(m)) => m.get(key), + _ => None, + } +} + +/// `{previous_utc, sections}`: what changed since the newest run kept in +/// `history/` that holds a summary.json (generate-report.sh keeps one per +/// run), or `None` without one. A previous run whose summary or manifest +/// cannot be read, or whose summary is not shaped as a summary, counts as +/// none. +fn new_since(dir: &Path, cur: &Value) -> Option { + let prev_dir = delta::newest_history(dir)?; + let bytes = std::fs::read(prev_dir.join("summary.json")).ok()?; + let prev = pycompat::json::loads(std::str::from_utf8(&bytes).ok()?).ok()?; + let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; + let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; + let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; + let previous = match &prev { + Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), + _ => Value::Null, + }; + let mut out = indexmap::IndexMap::new(); + out.insert("previous_utc".to_owned(), previous); + out.insert( + "sections".to_owned(), + Value::Array( + delta::changed_sections(&changes) + .into_iter() + .map(Value::Str) + .collect(), + ), + ); + Some(Value::Object(out)) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs new file mode 100644 index 000000000..e33709b92 --- /dev/null +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -0,0 +1,355 @@ +use super::super::group_c::{self, GROUP_C}; +use super::super::model::{GROUP_A, Spec}; +use super::super::{Group, Input, group_b}; +use super::*; +use crate::testutil::{Dir, run}; +use pycompat::json::{Value, loads}; + +fn input(d: &Dir) -> Input { + Input { + sample: "S".into(), + sample_dir: d.path("S"), + declared_sex: None, + assets: d.path("assets"), + group: Group::All, + tables: d.path("tables"), + } +} + +fn summary(d: &Dir) -> Value { + std::fs::create_dir_all(d.path("S")).unwrap(); + collect(&input(d)).unwrap() +} + +fn at<'a>(v: &'a Value, path: &[&str]) -> &'a Value { + path.iter().fold(v, |v, k| match v { + Value::Object(m) => m.get(*k).unwrap_or_else(|| panic!("no {k} in {path:?}")), + Value::Array(a) => &a[k.parse::().unwrap()], + _ => panic!("{path:?}"), + }) +} + +fn has(v: &Value, key: &str) -> bool { + matches!(v, Value::Object(m) if m.contains_key(key)) +} + +fn texts(v: &Value) -> Vec { + match v { + Value::Array(a) => a + .iter() + .map(|x| match x { + Value::Str(s) => s.clone(), + _ => panic!("{x:?}"), + }) + .collect(), + _ => panic!("{v:?}"), + } +} + +fn word(v: &Value) -> &str { + match v { + Value::Str(s) => s, + _ => panic!("{v:?}"), + } +} + +#[test] +fn the_order_holds_every_section_once() { + let mut keys: Vec<&str> = GROUP_A + .iter() + .chain(group_b::GROUP_B.iter()) + .chain(GROUP_C.iter()) + .map(|s: &Spec| s.key) + .collect(); + assert_eq!(keys.len(), ORDER.len()); + keys.sort_unstable(); + let mut order = ORDER.to_vec(); + order.sort_unstable(); + assert_eq!(keys, order); +} + +#[test] +fn an_empty_folder_lists_every_step_and_every_line() { + let d = Dir::new("post-empty"); + let v = summary(&d); + let Value::Object(m) = &v else { panic!() }; + let keys: Vec<&str> = m.keys().map(String::as_str).collect(); + assert_eq!( + keys, + [ + "schema_version", + "sample", + "generated_utc", + "run", + "databases", + "manifest", + "sections", + "cyp2d6", + "not_run", + "not_assessed" + ] + ); + let Value::Object(sections) = at(&v, &["sections"]) else { + panic!() + }; + assert_eq!(sections.keys().map(String::as_str).collect::>(), ORDER); + let not_run = texts(at(&v, &["not_run"])); + assert_eq!(not_run.len(), 45); + assert_eq!(not_run[0], "Coverage (mosdepth)"); + assert!(!not_run.contains(&"Reads kept by alignment".to_owned())); + assert!(!not_run.contains(&"Carrier list".to_owned())); + assert!(!not_run.contains(&"Couple carrier screen".to_owned())); + let lines = texts(at(&v, &["not_assessed"])); + let mut expected: Vec<&str> = NOT_ASSESSED.to_vec(); + expected.extend([ + PRS_NOT_ADJUSTED, + HBA_NOT_ASSESSED, + MEI_NOT_ASSESSED, + VNTYPER_NOT_ASSESSED, + MCA_NOT_ASSESSED, + ]); + assert_eq!(lines, expected); +} + +#[test] +fn a_failed_step_says_so_under_not_run() { + let d = Dir::new("post-failed"); + d.put( + "S/logs/run_status.tsv", + "meta\tstarted_epoch\t1577836800\nstep\t06\tfailed (exit 1)\nstep\t02\tfailed (exit 1)\n", + ); + let v = summary(&d); + let not_run = texts(at(&v, &["not_run"])); + assert!(not_run.contains(&"ClinVar screen (step 06 failed)".to_owned())); + // The read count is no step of its own, failed or not; the carrier + // list shares step 06 with ClinVar. + assert!(!not_run.iter().any(|x| x.starts_with("Reads kept"))); + assert!(!not_run.iter().any(|x| x.starts_with("Carrier list"))); + assert_eq!(word(at(&v, &["sections", "carrier", "state"])), "failed"); +} + +#[test] +fn the_lines_go_when_their_step_assessed_the_sample() { + let d = Dir::new("post-assessed"); + d.put( + "S/vntyper/S_vntyper.tsv", + "key\tvalue\nstatus\tnegative\n", + ); + d.put( + "S/paralogs/S_hba_copy_number.tsv", + "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\tfour\n", + ); + let v = summary(&d); + let lines = texts(at(&v, &["not_assessed"])); + let vntyper = at(&v, &["sections", "vntyper"]); + if word(at(vntyper, &["state"])) == "ok" && word(at(vntyper, &["data", "status"])) == "negative" { + assert!(!lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); + } else { + assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); + } + let parascopy = at(&v, &["sections", "parascopy"]); + if word(at(parascopy, &["state"])) == "ok" + && word(at(parascopy, &["data", "hba", "status"])) == "called" + { + assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); + } +} + +#[test] +fn a_stale_vntyper_reading_is_not_assessed() { + // Only an ok section counts for VNtyper; any other state leaves the line. + let mk = |state: State, status: &str| { + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "vntyper").unwrap(), + Ok(None), + None, + ); + s.state = state; + s.data = Some(Data::Json( + loads(&format!(r#"{{"status": "{status}"}}"#)).unwrap(), + )); + s + }; + let lines = not_assessed(&[mk(State::Ok, "positive")]); + assert!(!lines.contains(&VNTYPER_NOT_ASSESSED)); + let lines = not_assessed(&[mk(State::Stale, "positive")]); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED)); + let lines = not_assessed(&[mk(State::Ok, "failed")]); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED)); +} + +#[test] +fn paraphase_needs_both_hba_genes_called() { + let paralogs = |genes: &str| { + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "paralogs").unwrap(), + Ok(None), + None, + ); + s.state = State::Ok; + s.data = Some(Data::Json( + loads(&format!(r#"{{"genes": {genes}}}"#)).unwrap(), + )); + s + }; + let both = r#"[{"region": "hba", "gene": "HBA1", "status": "called"}, + {"region": "hba", "gene": "HBA2", "status": "called"}]"#; + let one = r#"[{"region": "hba", "gene": "HBA1", "status": "called"}, + {"region": "hba", "gene": "HBA2", "status": "no call"}]"#; + // A row without a region, the shape that stopped the Python collector. + let odd = r#"[{"gene": "HBA1", "status": "called"}, 7]"#; + assert!(!not_assessed(&[paralogs(both)]).contains(&HBA_NOT_ASSESSED)); + assert!(not_assessed(&[paralogs(one)]).contains(&HBA_NOT_ASSESSED)); + assert!(not_assessed(&[paralogs(odd)]).contains(&HBA_NOT_ASSESSED)); +} + +#[test] +fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { + // An error and a panic alike come back as the section's key and why. + assert_eq!(group_c::contain("cnvpytor", || Ok(())), None); + assert_eq!( + group_c::contain("expansions", || Err("no table".into())), + Some(("expansions", "no table".to_owned())) + ); + let panicked = group_c::contain("locityper", || panic!("planted")); + assert_eq!( + panicked, + Some(( + "locityper", + "internal error: the post pass of locityper stopped".to_owned() + )) + ); + // The section is then unreadable, with no data, and names its file. + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "locityper").unwrap(), + Ok(Some(( + "locityper/S_locityper.tsv".into(), + 1_600_000_000.0, + Data::Json(Value::Object(indexmap::IndexMap::new())), + ))), + None, + ); + s.make_unreadable("internal error: the post pass of locityper stopped"); + let j = s.to_json(); + assert_eq!(word(at(&j, &["state"])), "unreadable"); + assert_eq!( + word(at(&j, &["note"])), + "S_locityper.tsv: internal error: the post pass of locityper stopped" + ); + assert_eq!(at(&j, &["source"]), &Value::Null); + assert_eq!(at(&j, &["data"]), &Value::Object(indexmap::IndexMap::new())); +} + +#[test] +fn an_infinite_telomere_read_length_keeps_the_summary() { + // The Python collector stopped on this table with an OverflowError and + // wrote no summary.json; here only the telomere section is unreadable. + let d = Dir::new("post-inf"); + d.put( + "S/telomere/S/S/S_summary.tsv", + "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t1000\tinf\t40\t1.5\n", + ); + d.put( + "S/mosdepth/S.mosdepth.summary.txt", + "chrom\tlength\tbases\tmean\tmin\tmax\ntotal\t100\t3000\t30\t0\t60\n", + ); + let v = summary(&d); + assert_eq!(word(at(&v, &["sections", "telomere", "state"])), "unreadable"); + assert_eq!(word(at(&v, &["sections", "coverage", "state"])), "ok"); +} + +#[test] +fn a_declared_female_drops_cnvpytor_chry_calls_across_groups() { + // The declared sex comes from step 33's table (group a) and changes + // CNVpytor's count (group c). + let d = Dir::new("post-sex"); + d.put( + "S/qc/S_sample_qc.tsv", + "key\tvalue\nsample\tS\ndeclared_sex\tfemale\ninferred_sex\tfemale\nsex_check\tok\n", + ); + d.put( + "S/cnvpytor/S_cnvs.txt", + "deletion\tchrY:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n\ +deletion\tchr2:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n", + ); + let v = summary(&d); + assert_eq!(word(at(&v, &["run", "declared_sex"])), "female"); + let cnv = at(&v, &["sections", "cnvpytor"]); + if word(at(cnv, &["state"])) == "ok" { + assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); + assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); + } +} + +const PREV: &str = r#"{"schema_version": 2, "generated_utc": "2020-01-01 00:00 UTC", + "sections": {"coverage": {"state": "ok", "data": {}}, "clinvar": {"state": "missing", "data": {}}}}"#; + +#[test] +fn new_since_compares_with_the_newest_kept_run() { + let d = Dir::new("post-history"); + d.put("S/history/20200101T000000/summary.json", PREV); + // A newer folder without a summary.json is a half-copied one: passed over. + d.put("S/history/20990101T000000/run_manifest.tsv", "run\tx\ty\n"); + let v = summary(&d); + assert_eq!( + at(&v, &["new_since", "previous_utc"]), + &Value::Str("2020-01-01 00:00 UTC".into()) + ); + assert!(texts(at(&v, &["new_since", "sections"])).contains(&"coverage".to_owned())); +} + +#[test] +fn a_previous_run_that_cannot_be_read_leaves_new_since_out() { + for body in ["{not json", "[]", r#"{"sections": 7}"#] { + let d = Dir::new("post-history-bad"); + d.put("S/history/20200101T000000/summary.json", body); + let v = summary(&d); + assert!(!has(&v, "new_since"), "{body}"); + } +} + +#[test] +fn the_recheck_block_comes_from_its_state_file() { + let d = Dir::new("post-recheck"); + d.put( + "S/recheck/state.json", + r#"{"checked_utc": "2026-01-02 03:04:05 UTC", "items": [{"item": "clinvar", "class": "newer", "x": 1}, 5], + "affected_steps": ["06"]}"#, + ); + let v = summary(&d); + assert_eq!( + word(at(&v, &["recheck", "checked_utc"])), + "2026-01-02 03:04:05 UTC" + ); + assert_eq!(word(at(&v, &["recheck", "items", "0", "item"])), "clinvar"); + // A check older than the run's manifest is of an earlier run. + d.put( + "S/run_manifest.tsv", + "run\twritten_utc\t2027-01-01 00:00:00 UTC\n", + ); + assert!(!has(&summary(&d), "recheck")); +} + +#[test] +fn without_only_the_command_writes_summary_json_in_the_folder() { + let d = Dir::new("post-cli"); + std::fs::create_dir_all(d.path("S")).unwrap(); + let dir = d.path("S"); + let assets = d.path("assets"); + let (code, out, err) = run( + crate::collect::cli, + &["--sample", "S", "--sample-dir", &dir, "--assets", &assets], + ); + assert_eq!((code, err.as_str()), (0, "")); + let target = format!("{dir}/summary.json"); + assert_eq!(out, format!("Summary written: {target}\n")); + let v = loads(&d.read("S/summary.json")).unwrap(); + assert!(has(&v, "not_assessed")); + // A part of the sections never takes summary.json's place. + let (code, _, err) = run( + crate::collect::cli, + &["--sample", "S", "--sample-dir", &dir, "--only", "a"], + ); + assert_eq!(code, 2); + assert!(err.contains("--only needs --out"), "{err}"); +} diff --git a/rust/crates/og-summary/src/collect/tests.rs b/rust/crates/og-summary/src/collect/tests.rs index 75031f745..4f622c77f 100644 --- a/rust/crates/og-summary/src/collect/tests.rs +++ b/rust/crates/og-summary/src/collect/tests.rs @@ -448,22 +448,22 @@ fn the_command_line() { let text = d.read("out.json"); assert!(text.starts_with("{\n \"schema_version\": 2,\n \"sample\": \"S\",\n")); assert!(text.ends_with("}\n")); - // --only is required until every section is ported. + // Without --only it writes every section and the post pass. let (code, _, err) = run( cli, &["--sample", "S", "--sample-dir", &d.path("S"), "--out", &out], ); - assert_eq!(code, 2); - assert!(err.contains("required: --only"), "{err}"); - // So is --out: a part of the sections must not land on the sample's - // complete summary.json. + assert_eq!((code, err.as_str()), (0, "")); + assert!(d.read("out.json").contains("\n \"not_assessed\": [\n")); + // --only needs --out: a part of the sections must not land on the + // sample's complete summary.json. std::fs::write(d.path("S/summary.json"), "complete\n").unwrap(); let (code, _, err) = run( cli, &["--sample", "S", "--sample-dir", &d.path("S"), "--only", "a"], ); assert_eq!(code, 2); - assert!(err.contains("required: --out"), "{err}"); + assert!(err.contains("--only needs --out"), "{err}"); assert_eq!(d.read("S/summary.json"), "complete\n"); let (code, _, err) = run( cli, From 5e22114a3ad3a25009b7e76bf3211631d567d375 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:34:15 +0100 Subject: [PATCH 11/86] test(parity): compare the whole summary.json with the Python collector Nine cases run og summary collect without --only against bin/collect_summary.py on folders that join the three groups' fixtures, with failed steps, an empty folder, kept runs in history/ and a database re-check. A telomere table with an infinite read length stopped the Python with an uncaught OverflowError; og keeps the summary with that section unreadable (the all-inf golden). Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect.rs | 3 +- rust/crates/og-summary/src/collect/group_c.rs | 5 +- rust/crates/og-summary/src/collect/post.rs | 16 +- .../og-summary/src/collect/post/tests.rs | 56 +- tests/parity/cases.tsv | 9 + tests/parity/gen/summary-collect-full.sh | 107 +++ .../summary-collect/all-inf/all-inf.json | 820 ++++++++++++++++++ .../golden/summary-collect/all-inf/exit | 1 + .../golden/summary-collect/all-inf/stderr | 0 .../golden/summary-collect/all-inf/stdout | 1 + 10 files changed, 981 insertions(+), 37 deletions(-) create mode 100644 tests/parity/gen/summary-collect-full.sh create mode 100644 tests/parity/golden/summary-collect/all-inf/all-inf.json create mode 100644 tests/parity/golden/summary-collect/all-inf/exit create mode 100644 tests/parity/golden/summary-collect/all-inf/stderr create mode 100644 tests/parity/golden/summary-collect/all-inf/stdout diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 13b22eb76..4bbc9529f 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -51,8 +51,7 @@ use sections::{Callability, Coverage, Data, Mosaic, ReadCount, SampleQc, SexChec use std::io::Write; use std::path::Path; -const USAGE: &str = - "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] +const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] [--declared-sex DECLARED_SEX] [--assets ASSETS] [--only {a,b,c}] "; diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 3450f6b5d..29f66b870 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -295,7 +295,10 @@ pub fn contain( match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), - Err(_) => Some((key, format!("internal error: the post pass of {key} stopped"))), + Err(_) => Some(( + key, + format!("internal error: the post pass of {key} stopped"), + )), } } diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 42b9fe1ad..54c5c36da 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -139,9 +139,7 @@ pub fn collect(input: &Input) -> Result { }) .collect() }; - let a = a - .join() - .unwrap_or_else(|_| stopped(&super::model::GROUP_A)); + let a = a.join().unwrap_or_else(|_| stopped(&super::model::GROUP_A)); let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); (a, b, c) }); @@ -191,10 +189,10 @@ pub fn collect(input: &Input) -> Result { if let Some(r) = delta::read_recheck(&dir.join("recheck").join("state.json")) { m.insert("recheck".into(), Value::Object(r)); } - if let Some(n) = new_since(dir, &v) { - if let Value::Object(m) = &mut v { - m.insert("new_since".into(), n); - } + if let Some(n) = new_since(dir, &v) + && let Value::Object(m) = &mut v + { + m.insert("new_since".into(), n); } Ok(v) } @@ -216,7 +214,9 @@ fn others(b: &[Section]) -> Others { cyrius: sec("cyrius") .filter(|s| live(s)) .and_then(|s| match &s.data { - Some(Data::Cyrius(c)) => Some((s.state, Some(c.genotype.clone()), c.filter.clone())), + Some(Data::Cyrius(c)) => { + Some((s.state, Some(c.genotype.clone()), c.filter.clone())) + } _ => None, }), } diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs index e33709b92..bcf38afd5 100644 --- a/rust/crates/og-summary/src/collect/post/tests.rs +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -92,7 +92,10 @@ fn an_empty_folder_lists_every_step_and_every_line() { let Value::Object(sections) = at(&v, &["sections"]) else { panic!() }; - assert_eq!(sections.keys().map(String::as_str).collect::>(), ORDER); + assert_eq!( + sections.keys().map(String::as_str).collect::>(), + ORDER + ); let not_run = texts(at(&v, &["not_run"])); assert_eq!(not_run.len(), 45); assert_eq!(not_run[0], "Coverage (mosdepth)"); @@ -129,30 +132,17 @@ fn a_failed_step_says_so_under_not_run() { } #[test] -fn the_lines_go_when_their_step_assessed_the_sample() { +fn step_35s_hba_count_assesses_hba() { let d = Dir::new("post-assessed"); - d.put( - "S/vntyper/S_vntyper.tsv", - "key\tvalue\nstatus\tnegative\n", - ); d.put( "S/paralogs/S_hba_copy_number.tsv", "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\tfour\n", ); let v = summary(&d); + assert_eq!(word(at(&v, &["sections", "parascopy", "state"])), "ok"); let lines = texts(at(&v, &["not_assessed"])); - let vntyper = at(&v, &["sections", "vntyper"]); - if word(at(vntyper, &["state"])) == "ok" && word(at(vntyper, &["data", "status"])) == "negative" { - assert!(!lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); - } else { - assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); - } - let parascopy = at(&v, &["sections", "parascopy"]); - if word(at(parascopy, &["state"])) == "ok" - && word(at(parascopy, &["data", "hba", "status"])) == "called" - { - assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); - } + assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); } #[test] @@ -160,7 +150,11 @@ fn a_stale_vntyper_reading_is_not_assessed() { // Only an ok section counts for VNtyper; any other state leaves the line. let mk = |state: State, status: &str| { let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "vntyper").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "vntyper") + .unwrap(), Ok(None), None, ); @@ -182,7 +176,11 @@ fn a_stale_vntyper_reading_is_not_assessed() { fn paraphase_needs_both_hba_genes_called() { let paralogs = |genes: &str| { let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "paralogs").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "paralogs") + .unwrap(), Ok(None), None, ); @@ -221,7 +219,11 @@ fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { ); // The section is then unreadable, with no data, and names its file. let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "locityper").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "locityper") + .unwrap(), Ok(Some(( "locityper/S_locityper.tsv".into(), 1_600_000_000.0, @@ -254,7 +256,10 @@ fn an_infinite_telomere_read_length_keeps_the_summary() { "chrom\tlength\tbases\tmean\tmin\tmax\ntotal\t100\t3000\t30\t0\t60\n", ); let v = summary(&d); - assert_eq!(word(at(&v, &["sections", "telomere", "state"])), "unreadable"); + assert_eq!( + word(at(&v, &["sections", "telomere", "state"])), + "unreadable" + ); assert_eq!(word(at(&v, &["sections", "coverage", "state"])), "ok"); } @@ -275,10 +280,9 @@ deletion\tchr2:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n" let v = summary(&d); assert_eq!(word(at(&v, &["run", "declared_sex"])), "female"); let cnv = at(&v, &["sections", "cnvpytor"]); - if word(at(cnv, &["state"])) == "ok" { - assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); - assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); - } + assert_eq!(word(at(cnv, &["state"])), "ok"); + assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); + assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); } const PREV: &str = r#"{"schema_version": 2, "generated_utc": "2020-01-01 00:00 UTC", diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 7c21916ad..14edaa05e 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -870,6 +870,15 @@ summary-collect cb-notbi gen:summary-clinical.sh,summary-collect/oracle.sh,summa summary-collect cb-notgz gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notgz --assets assets --out cb-notgz.json --only b cb-notgz.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-empty gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-empty --assets assets --out cb-empty.json --only b cb-empty.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-no-assets gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets nowhere --out cb-no-assets.json --only b cb-no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect all gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir all --assets assets --out all.json all.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-nf gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir nf --assets assets --out all-nf.json all-nf.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-bad gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir bad --assets assets --out all-bad.json all-bad.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-failed gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir failed --assets assets --out all-failed.json all-failed.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-empty gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir empty --assets assets --out all-empty.json all-empty.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-history gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history --assets assets --out all-history.json all-history.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-history-bad gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history-bad --assets assets --out all-history-bad.json all-history-bad.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-inf gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir inf --assets assets --out all-inf.json all-inf.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-default-out gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history-bad --assets assets history-bad/summary.json json:mask=generated_utc PYTHON_IMAGE y-tree-positions small gen:y-inputs.sh --tree-dir t pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions big gen:y-inputs.sh --tree-dir ./big/ pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions odd gen:y-inputs.sh --tree-dir odd pos.txt pos.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/summary-collect-full.sh b/tests/parity/gen/summary-collect-full.sh new file mode 100644 index 000000000..94d4db167 --- /dev/null +++ b/tests/parity/gen/summary-collect-full.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# summary-collect-full.sh: writes the sample folders of the whole-file +# summary-collect parity cases (og summary collect without --only) in the +# current directory, the same every time. It runs the generators of the +# three groups (summary-collect.sh, summary-clinical.sh and +# summary-collect-c.sh) and joins their folders, so one folder holds every +# section and the post pass reads across the groups: the repeat readings +# with the CPIC state, Locityper's primary calls with pypgx and Cyrius, +# CNVpytor's chrY filter with step 33's declared sex, the cyp2d6 block, +# not_run and not_assessed. On top: a run with failed steps, an empty +# folder, previous runs kept in history/ (one half-copied, one whose +# summary is no JSON) with a database re-check, and a telomere table whose +# infinite read length stopped the Python collector (a golden: og keeps +# the summary and marks that one section unreadable). +# py/ holds the modules the oracle imports at the oracle's commit, with the +# real run_delta.py for recheck and new_since. The case stages +# cb_vep.vcf.gz and its index beside the folders. +# scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C TZ=UTC +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "${here}/../../.." && pwd) +ORACLE_SHA=98aa9252ff77825a8a797d388b449931fa9a234e + +put() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" > "$1"; at "$NEW" "$1"; } +at() { touch -t "$1" "${@:2}"; } # [[CC]YY]MMDDhhmm[.SS], in UTC +NEW=202103040506.07 # after the run of the three groups' folders + +for g in a b c; do mkdir -p "parts/$g"; done +(cd parts/a && bash "${here}/summary-collect.sh") +(cd parts/b && bash "${here}/summary-clinical.sh") +(cd parts/c && bash "${here}/summary-collect-c.sh") + +# merge DEST SRC...: the folders' files, the later folder's file winning; +# times and symlinks kept. +merge() { + local dest=$1 src + shift + mkdir -p "$dest" + for src in "$@"; do cp -pPR "parts/${src}/." "$dest"/; done +} + +mkdir -p assets py pymods +for g in a b c; do cp -pR "parts/${g}/assets/." assets/; done +cp -pR parts/b/pymods/. pymods/ +cp -pR parts/c/py/. py/ +git -C "$root" cat-file -e "${ORACLE_SHA}^{commit}" 2> /dev/null \ + || git -C "$root" fetch -q --depth=1 origin "$ORACLE_SHA" +git -C "$root" show "${ORACLE_SHA}:bin/run_delta.py" > py/run_delta.py + +# all: every section of the three groups, the bash names. +merge all a/full b/cb-full c/full +# nf: the Nextflow names, steps failed and from an earlier run. +merge nf a/nf b/cb-nf c/nf +# bad: an unreadable file in sections of each group. +merge bad a/bad-a b/cb-bad c/bad1 +# failed: steps that failed and wrote nothing, under not_run. +merge failed a/failed b/cb-failed +mkdir -p empty + +# history: all, with three earlier runs kept. The newest has no summary +# (a half-copied folder) and is passed over; the next is compared: its +# ClinVar, CPIC, pypgx and PRS results and its section states differ, and +# so do the inputs of its manifest. A database re-check newer than the +# run's manifest. +merge history a/full b/cb-full c/full +put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" +put history/history/20200101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2019-12-01 00:00 UTC", "sections": {}}' +put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2021-01-01 00:00 UTC", + "sections": { + "coverage": {"title": "Coverage (mosdepth)", "state": "missing", "data": {}}, + "clinvar": {"title": "ClinVar screen", "state": "ok", "data": {"hits": [ + {"chrom": "chr1", "pos": "45331556", "ref": "C", "alt": "T", "gene": "MUTYH", "significance": "Likely_pathogenic", "stars": 1, "review_status": "x", "genotype": "1/1"}, + {"chrom": "chr9", "pos": "1", "ref": "A", "alt": "G", "gene": "X", "significance": "Pathogenic", "stars": 2}]}}, + "cpic": {"title": "CPIC", "state": "ok", "data": {"genes": [{"gene": "CYP2D6", "diplotype": "*1/*1", "phenotype": "Normal"}]}}, + "pypgx": {"title": "pypgx", "state": "ok", "data": {"cyp2d6": "*1/*1"}}, + "prs": {"title": "PRS", "state": "ok", "data": {"scores": [{"pgs_id": "PGS000001", "condition": "x", "percentile": 10}]}}, + "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "data": {}}}, + "cyp2d6": {"consensus": {"result": "*1/*1"}}}\n' +put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" +mkdir -p history/history/20220101T000000 +put history/history/20220101T000000/run_manifest.tsv "run\twritten_utc\t2022-01-01 00:00:00 UTC\n" +put history/recheck/state.json '{"checked_utc": "2021-03-05 00:00:00 UTC", "items": [ + {"item": "clinvar", "label": "ClinVar", "class": "newer-release", "ran_with": "2021-03-01", "upstream": "2021-03-04", "action": "rerun step 06", "extra": 1}, + {"item": "vep", "label": "VEP", "class": "no-newer-release"}, 7], + "affected_steps": ["06"], "rerun": {"command": "run-all.sh"}}\n' + +# history-bad: the only kept run has a summary that is no JSON, so nothing +# is compared; a re-check older than the run's manifest is of an earlier +# run and left out. +mkdir -p history-bad +put history-bad/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\n" +put history-bad/history/20210101T000000/summary.json '{"schema_version": 2,' +put history-bad/recheck/state.json '{"checked_utc": "2021-03-01 00:00:00 UTC", "items": []}\n' + +# inf: a telomere table whose read length is infinite, beside the +# coverage, step 33 and repeat sections. The Python collector stopped with +# an uncaught OverflowError and wrote no summary; og writes it with the +# telomere section unreadable (a golden). +mkdir -p inf +for f in mosdepth/S.mosdepth.summary.txt qc/S_sample_qc.tsv logs/run_status.tsv; do + mkdir -p "inf/$(dirname "$f")" && cp -p "parts/a/full/$f" "inf/$f" +done +cp -pR parts/c/full/expansion_hunter inf/ +put inf/telomere/S/S/S_summary.tsv "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t1000\tinf\t40\t1.5\n" + +rm -rf parts diff --git a/tests/parity/golden/summary-collect/all-inf/all-inf.json b/tests/parity/golden/summary-collect/all-inf/all-inf.json new file mode 100644 index 000000000..99b10ea8d --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/all-inf.json @@ -0,0 +1,820 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-13 23:22 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). Not compared with PharmCAT (no unambiguous UGT1A1 call from step 27)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "OverflowError: cannot round", + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "SMN1/SMN2 copy number (Parascopy, short reads)", + "Paralog genes (Paraphase, PacBio HiFi)", + "HLA typing (T1K)", + "Polygenic risk scores", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge", + "Structural variants, class 4 and 5 (AnnotSV)", + "Runs of homozygosity", + "Mitochondrial haplogroup", + "Y-chromosome haplogroup (Yleaf)", + "Y placement on the current public tree", + "Y-STR markers (HipSTR)", + "Raw-data export (23andMe v5 layout)", + "Blood groups (predicted)", + "Lp(a): LPA KIV-2 repeat count", + "MUC1 VNTR frameshifts (VNtyper)", + "Locityper second opinion at hard loci", + "Mitochondrial variants", + "Traits panel", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)", + "Viral DNA in the unmapped reads", + "Symptom-driven ranking (Exomiser)", + "Clonal haematopoiesis (CHIP)", + "Mobile element insertions (MEGAnE)", + "Mosaic chromosomal alterations (MoChA)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): not looked for (step 50, MEGAnE, is opt-in)", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all-inf/exit b/tests/parity/golden/summary-collect/all-inf/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/exit @@ -0,0 +1 @@ +0 diff --git a/tests/parity/golden/summary-collect/all-inf/stderr b/tests/parity/golden/summary-collect/all-inf/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/summary-collect/all-inf/stdout b/tests/parity/golden/summary-collect/all-inf/stdout new file mode 100644 index 000000000..437515d43 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/stdout @@ -0,0 +1 @@ +Summary written: all-inf.json From 88dd793198d73798bb50013332594a1bc61a1031 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:04:00 +0100 Subject: [PATCH 12/86] docs(summary): og summary collect writes the whole summary.json Bead: pgp-bs9u.44 --- CHANGELOG.md | 3 +++ docs/rust.md | 2 +- rust/README.md | 2 +- rust/og/src/main.rs | 2 +- tests/parity/inputs/summary-collect/oracle.sh | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7489faaa..0a14652ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust `og` binary now writes the whole summary.json: `og summary collect` without `--only` reads all 48 sections over one walk of the sample folder, then runs the post pass. The post pass joins sections: the repeat readings with the traits and CPIC states, Locityper's primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter with the declared sex. It also builds the CYP2D6 block and writes `not_run`, `not_assessed`, `recheck` and `new_since`. Without `--out` it writes `summary.json` in the sample folder, as the Python did; `--only` still needs `--out`. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The whole file is the same as the Python's on 8 of 9 new parity cases, compared as JSON with key order and only `generated_utc` masked; the ninth holds the deliberate fix below. On a synthetic folder with every section it takes 0.01 to 0.03 s and 15 MB, where the Python took 0.15 to 0.21 s and 42 to 44 MB. With a 1,000,000-record VCF it takes 0.44 to 0.62 s and 17 to 20 MB, where the Python took 1.5 s and 42 to 44 MB. What differs from the Python: + - A part of the post pass that fails makes only its own section unreadable, with a note that names the file. The Python ran the post pass outside its per-section error handling, so one failure there lost summary.json and both reports. + - A section reader that stops on an error the Python did not catch keeps the summary too. A telomere table with an infinite read length stopped the Python with an OverflowError and no summary.json; og marks only the telomere section unreadable. - Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published. A staging failure or a directory destination leaves both previous tables intact. Each file is then replaced by a separate atomic rename. The pair is not a transaction: a failure of the second rename can leave the first table updated. Catalogs and output remain plain TSV; quotes never hide a tab or bypass validation. - The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. diff --git a/docs/rust.md b/docs/rust.md index 57471737d..528d0a63e 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -21,7 +21,7 @@ rust/ crates/og-qc/ og qc sex-loss: mosaic loss of Y or X for step 33 crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) - crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect --only a|b|c, QC, clinical and the remaining summary sections (no caller yet); og roh summary (step 11) + crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass, or one group with --only a|b|c (no caller yet); og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read diff --git a/rust/README.md b/rust/README.md index 449ce5867..14724865c 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 6b4833c7b..82e29d218 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -50,7 +50,7 @@ const USAGE: &str = "usage: og [args] og prs format [args] og prs table [args] og mito annotate|deletions|copies [args] - og summary collect --only {a,b,c} [args] + og summary collect [--only {a,b,c}] [args] og roh summary --sample SAMPLE --roh ROH [--out OUT] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] diff --git a/tests/parity/inputs/summary-collect/oracle.sh b/tests/parity/inputs/summary-collect/oracle.sh index 3e33b6992..7add40e40 100644 --- a/tests/parity/inputs/summary-collect/oracle.sh +++ b/tests/parity/inputs/summary-collect/oracle.sh @@ -6,8 +6,8 @@ # --only is og's alone (the Python writes every section, and the case # compares the group's subtrees). The modules collect_summary imports are # stubbed, except those the group b and c generators stage in pymods/ -# and py/. No other module is reached for these sample folders, except -# run_delta's two readers of history/ and recheck/, which find nothing here. +# and py/. run_delta is stubbed to find no history/ and no recheck/, unless +# py/ brings the real one (the whole-file cases, summary-collect-full.sh). prog=$1 shift d=$(mktemp -d) From 888cd36fcdb34eeba513bb77fbec943e763997e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:22 +0100 Subject: [PATCH 13/86] test(summary): plant four post-pass faults (red control, reverted next) not_run keeps the sections that are no step, a stale VNtyper reading counts as assessed, new_since takes the current run's date, and a panic in a post-pass part is no longer contained. The next commit removes them. Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect/group_c.rs | 3 ++- rust/crates/og-summary/src/collect/post.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 29f66b870..dcc7380fd 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -292,7 +292,8 @@ pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { + // Planted fault (red control): the panic is no longer caught. + match Ok::<_, ()>(part()) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), Err(_) => Some(( diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 54c5c36da..faec5a064 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -234,7 +234,7 @@ fn not_run(sections: &[Section]) -> Vec { sections .iter() .filter(|s| matches!(s.state, State::Missing | State::Failed)) - .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) + .filter(|s| !NOT_A_STEP[..0].contains(&s.spec.key)) // planted fault (red control) .map(|s| { if s.state == State::Failed { format!("{} (step {} failed)", s.spec.title, s.spec.step) @@ -279,7 +279,7 @@ fn not_assessed(sections: &[Section]) -> Vec<&'static str> { if state("mei") == State::Missing { out.push(MEI_NOT_ASSESSED); } - let vntyper_read = state("vntyper") == State::Ok + let vntyper_read = matches!(state("vntyper"), State::Ok | State::Stale) && ["positive", "possible", "negative"] .iter() .any(|w| is(field("vntyper", "status"), w)); @@ -317,7 +317,7 @@ fn new_since(dir: &Path, cur: &Value) -> Option { let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; - let previous = match &prev { + let previous = match cur { Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), _ => Value::Null, }; From af005e271db960a44135b5208b8a587b5496e515 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:55 +0100 Subject: [PATCH 14/86] test(summary): remove the planted post-pass faults Reverts 888cd36f, the red control of this PR. Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect/group_c.rs | 3 +-- rust/crates/og-summary/src/collect/post.rs | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index dcc7380fd..29f66b870 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -292,8 +292,7 @@ pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - // Planted fault (red control): the panic is no longer caught. - match Ok::<_, ()>(part()) { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), Err(_) => Some(( diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index faec5a064..54c5c36da 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -234,7 +234,7 @@ fn not_run(sections: &[Section]) -> Vec { sections .iter() .filter(|s| matches!(s.state, State::Missing | State::Failed)) - .filter(|s| !NOT_A_STEP[..0].contains(&s.spec.key)) // planted fault (red control) + .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) .map(|s| { if s.state == State::Failed { format!("{} (step {} failed)", s.spec.title, s.spec.step) @@ -279,7 +279,7 @@ fn not_assessed(sections: &[Section]) -> Vec<&'static str> { if state("mei") == State::Missing { out.push(MEI_NOT_ASSESSED); } - let vntyper_read = matches!(state("vntyper"), State::Ok | State::Stale) + let vntyper_read = state("vntyper") == State::Ok && ["positive", "possible", "negative"] .iter() .any(|w| is(field("vntyper", "status"), w)); @@ -317,7 +317,7 @@ fn new_since(dir: &Path, cur: &Value) -> Option { let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; - let previous = match cur { + let previous = match &prev { Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), _ => Value::Null, }; From 7b0a8e735331d3e7e8411ec670bcb9918b38c079 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:23:59 +0100 Subject: [PATCH 15/86] test(parity): name no image in the whole-summary fixture manifests The images check reads any image name in a tracked file as one that bypasses versions.env; the fixture's DEEPVARIANT_IMAGE rows now hold plain version text. Bead: pgp-bs9u.44 --- tests/parity/gen/summary-collect-full.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parity/gen/summary-collect-full.sh b/tests/parity/gen/summary-collect-full.sh index 94d4db167..accdcdf73 100644 --- a/tests/parity/gen/summary-collect-full.sh +++ b/tests/parity/gen/summary-collect-full.sh @@ -64,7 +64,7 @@ mkdir -p empty # so do the inputs of its manifest. A database re-check newer than the # run's manifest. merge history a/full b/cb-full c/full -put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" +put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tv1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" put history/history/20200101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2019-12-01 00:00 UTC", "sections": {}}' put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2021-01-01 00:00 UTC", "sections": { @@ -77,7 +77,7 @@ put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample" "prs": {"title": "PRS", "state": "ok", "data": {"scores": [{"pgs_id": "PGS000001", "condition": "x", "percentile": 10}]}}, "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "data": {}}}, "cyp2d6": {"consensus": {"result": "*1/*1"}}}\n' -put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" +put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tv1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" mkdir -p history/history/20220101T000000 put history/history/20220101T000000/run_manifest.tsv "run\twritten_utc\t2022-01-01 00:00:00 UTC\n" put history/recheck/state.json '{"checked_utc": "2021-03-05 00:00:00 UTC", "items": [ From 8dd88d6bcc29808a3454e7743d514a8c4965e530 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:46:19 +0100 Subject: [PATCH 16/86] docs(ancestry): give the archaic table's measured cost on the real table Chromosomes 1 to 6 and 21 of the installed table, rebuilt by og from the same public inputs, equal the Python-built table after decompression. The synthetic table figure understated mask memory, so the changelog now gives the measured chromosome 1 cost. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d74fe2a71..30fe87bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On the real archaic table, `og archaic table` merges chromosome 1 (4.1 million sites) in 18 s and 190 MB where the Python took 76 s and 1.25 GB; bcftools and Picard still take most of each chromosome's setup. On a synthetic input of real size, step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: + - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes (chromosomes 1 to 6 and 21 of the real table, rebuilt from the same public files, are the same after decompression), so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. From a3f08e08baa9a849eac0643faa8ea92f9e1e91f0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:36:49 +0100 Subject: [PATCH 17/86] fix(summary): claim only error containment in the post pass The release profile aborts on a panic, so catch_unwind in contain() caught nothing in the shipped og. contain() now maps the error a part returns, and the docs say the parts read with defaults instead of promising to catch a panic. --- rust/crates/og-summary/src/collect/group_c.rs | 17 ++++++----------- rust/crates/og-summary/src/collect/post.rs | 8 +++++--- .../crates/og-summary/src/collect/post/tests.rs | 17 +++-------------- rust/crates/og-summary/src/lib.rs | 4 ++-- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 29f66b870..d9c8c9506 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -238,8 +238,10 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { /// order, each with its state and data): the repeat readings with the /// traits and CPIC states, each Locityper locus's primary call, and the /// chrY filter of CNVpytor's count. Each part runs on its own: one that -/// fails (an error or a panic) is returned as its section's key and why, -/// and the others still run. +/// returns an error is returned as its section's key and why, and the +/// others still run. The parts read with defaults, so a row of an +/// unexpected shape is no error. A panic is not caught: the release binary +/// aborts on one (`panic = "abort"`). pub fn post( c: &Ctx<'_>, sections: &mut [(Spec, State, Option<&mut Data>)], @@ -287,19 +289,12 @@ pub fn post( } /// Runs one part of the post pass: `None` when it went through, else its -/// section's key and why it failed, for an error and for a panic alike. +/// section's key and the error it returned. pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { - Ok(Ok(())) => None, - Ok(Err(e)) => Some((key, e)), - Err(_) => Some(( - key, - format!("internal error: the post pass of {key} stopped"), - )), - } + part().err().map(|e| (key, e)) } fn json(d: &Data) -> Option<&Value> { diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 54c5c36da..6065d5402 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -13,9 +13,11 @@ //! - `recheck`: the last database re-check of the sample; //! - `new_since`: what changed since the previous run kept in `history/`. //! -//! Each part that changes a section runs on its own: when it fails, only -//! its section becomes `unreadable`, with a note naming the file, and the -//! summary is still written. The Python collector ran this pass outside +//! Each part that changes a section runs on its own: when it returns an +//! error, only its section becomes `unreadable`, with a note naming the +//! file, and the summary is still written. The parts read with defaults, so +//! a row of an unexpected shape is no error; a panic is not caught, as the +//! release binary aborts on one. The Python collector ran this pass outside //! its per-section error handling, so one failure there lost summary.json //! and both reports. A previous run that cannot be read leaves `new_since` //! out, as before; the comparison itself has no error to hide. diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs index bcf38afd5..34d18cc61 100644 --- a/rust/crates/og-summary/src/collect/post/tests.rs +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -203,20 +203,12 @@ fn paraphase_needs_both_hba_genes_called() { #[test] fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { - // An error and a panic alike come back as the section's key and why. + // An error comes back as the section's key and why. assert_eq!(group_c::contain("cnvpytor", || Ok(())), None); assert_eq!( group_c::contain("expansions", || Err("no table".into())), Some(("expansions", "no table".to_owned())) ); - let panicked = group_c::contain("locityper", || panic!("planted")); - assert_eq!( - panicked, - Some(( - "locityper", - "internal error: the post pass of locityper stopped".to_owned() - )) - ); // The section is then unreadable, with no data, and names its file. let mut s = Section::new( GROUP_C @@ -231,13 +223,10 @@ fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { ))), None, ); - s.make_unreadable("internal error: the post pass of locityper stopped"); + s.make_unreadable("no table"); let j = s.to_json(); assert_eq!(word(at(&j, &["state"])), "unreadable"); - assert_eq!( - word(at(&j, &["note"])), - "S_locityper.tsv: internal error: the post pass of locityper stopped" - ); + assert_eq!(word(at(&j, &["note"])), "S_locityper.tsv: no table"); assert_eq!(at(&j, &["source"]), &Value::Null); assert_eq!(at(&j, &["data"]), &Value::Object(indexmap::IndexMap::new())); } diff --git a/rust/crates/og-summary/src/lib.rs b/rust/crates/og-summary/src/lib.rs index ac4040411..4aa53eeca 100644 --- a/rust/crates/og-summary/src/lib.rs +++ b/rust/crates/og-summary/src/lib.rs @@ -7,8 +7,8 @@ //! table from pgsc_calc's output (`prs/_prs_summary.tsv`, //! `ancestry/_ancestry.tsv`). //! - [`mito`]: `og mito annotate|deletions|copies`, step 20's chrM tables. -//! - [`collect`]: `og summary collect`, summary.json, ported one group of -//! sections at a time beside the Python collector (`--only a`). +//! - [`collect`]: `og summary collect`, the whole summary.json, or one +//! group with `--only a|b|c`, beside the Python collector (no caller yet). //! //! Each tool exits 0 when it wrote its table, 1 with `ERROR: ` //! when an input cannot be read or the table cannot be written, and 2 with From fe1542c5908c452e42f8cc9d7a3ab197820a662a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:23:42 +0100 Subject: [PATCH 18/86] fix(ancestry): keep the previous archaic outputs when a run stops, and refuse bad report inputs og archaic genotypes now stages samples.txt, every genotype file and every mask, and publishes them only when the whole run has passed. A chromosome with no row deletes its old genotype file only on success. The VCF order check compares each record with the last record handed to the join, so a skipped record of another chromosome cannot hide an unsorted return. og ancient report stops on a non-finite eigenvalue or person PC, and on an ancient row with fewer PCs than the distance uses. The parity goldens of the four failing genotypes cases now expect the previous outputs to stay (no new file). --- CHANGELOG.md | 5 +- rust/crates/og-ancestry/src/ancient.rs | 120 +++++++++- rust/crates/og-ancestry/src/archaic.rs | 209 +++++++++++++++--- .../bad-code/gt/samples.txt.absent | 0 .../comes-back/gt/chr21.Altai.mask.bed.absent | 0 .../gt/chr21.Denisova.mask.bed.absent | 0 .../gt/chr21.Vindija.mask.bed.absent | 0 .../comes-back/gt/chr21.gt.gz.absent | 0 .../comes-back/gt/chr22.Altai.mask.bed.absent | 0 .../gt/chr22.Denisova.mask.bed.absent | 0 .../gt/chr22.Vindija.mask.bed.absent | 0 .../comes-back/gt/chr22.gt.gz.absent | 0 .../comes-back/gt/samples.txt.absent | 0 .../no-table-chrom/gt/samples.txt.absent | 0 .../no-table-chrom/gt/spans.tsv.absent | 0 .../two-samples/gt/samples.txt.absent | 0 16 files changed, 302 insertions(+), 32 deletions(-) create mode 100644 tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent create mode 100644 tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a01e5d8..e30598b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,12 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On the real archaic table, `og archaic table` merges chromosome 1 (4.1 million sites) in 18 s and 190 MB where the Python took 76 s and 1.25 GB; bcftools and Picard still take most of each chromosome's setup. On a synthetic input of real size, step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes (chromosomes 1 to 6 and 21 of the real table, rebuilt from the same public files, are the same after decompression), so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. - - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. + - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2, also when a skipped record (an indel, ALT `.`) of another chromosome sits between the two. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. + - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. `og archaic genotypes` stages `samples.txt`, every genotype file and every mask, and publishes them only when the whole run has passed: a run that stops leaves the previous set as it was, where the Python had already rewritten `samples.txt`, earlier chromosomes and, for a chromosome with no row, deleted the old genotype file. The renames are one at a time, so the set is not one transaction. - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. - Malformed input stops with one `ERROR:` line, not a Python traceback. + - `og ancient report` stops when an eigenvalue or the person's PCs in the distance are not finite (NaN or infinity), and when an ancient group's row holds fewer PCs than the distance uses. The Python wrote NaN distances, or ranked that group on fewer PCs. - `sites_genotyped` in step 44's stats still counts a position twice when the AADR holds two SNPs there, as the Python did; the genotypes themselves are right. - CI evidence rejects FASTA offset overflow and offsets outside the file before reading or publishing. Guard tests this in both debug and release builds. Every new dev verb fails on stdout or stderr write or flush errors. A truth VCF header after the first data record fails without replacing the destination. Additional deliberate refusals are documented and tested: zero-byte gzip inputs, zero-width unused FAI rows, non-ASCII or out-of-range annotation limits, non-writable staging parents, and gzip FASTA files with long FNAME headers. Publication replaces a file and leaves other hard links on the old inode. diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs index 6a3b6fc52..3693dec13 100644 --- a/rust/crates/og-ancestry/src/ancient.rs +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -1063,9 +1063,10 @@ fn nearest_populations( let mut rows = Vec::with_capacity(order.len()); for g in order { let members = &groups[g]; - let k = members[0].0.len().min(n_pcs); - let mut centre = Vec::with_capacity(k); - for i in 0..k { + // Every member must hold all n_pcs PCs: a shorter row would rank + // its group on fewer PCs than the person's distance claims. + let mut centre = Vec::with_capacity(n_pcs); + for i in 0..n_pcs { let mut xs = Vec::with_capacity(members.len()); for (pcs, _) in members { xs.push(*pcs.get(i).ok_or_else(|| { @@ -1348,6 +1349,19 @@ fn cmd_report(a: &Args, out: &mut dyn Write) -> Result<()> { let n_pcs = (DISTANCE_PCS as usize) .min(panel.evals.len()) .min(pcs.len()); + // NaN or infinity in the eigenvalues or the person's PCs would make + // every distance and plotted point meaningless: stop instead. + let not_finite = |path: &str, what: &str| { + Fail(format!( + "{path}: {what} holds a value that is not a finite number" + )) + }; + if panel.evals.iter().take(n_pcs).any(|v| !v.is_finite()) { + return Err(not_finite(pp, "the #eigvals: line")); + } + if pcs.iter().take(n_pcs.max(2)).any(|v| !v.is_finite()) { + return Err(not_finite(rp, &format!("the row of {sample}"))); + } let signs = axis_signs(&panel, pp, &run, rp, n_pcs.max(2))?; let person: Vec = signs.iter().zip(pcs).map(|(s, v)| s * v).collect(); let ancients = read_ancients(a.req("--ancients"))?; @@ -1569,4 +1583,104 @@ mod tests { assert_eq!(clean(" a\t b "), "a b"); assert_eq!(clean(" \t "), "."); } + + fn evec(evals: &[f64], rows: &[(&str, &str, &[f64])]) -> Evec { + let mut e = Evec { + evals: evals.to_vec(), + rows: Vec::new(), + index: HashMap::new(), + }; + for (id, label, pcs) in rows { + e.index.insert((*id).to_owned(), e.rows.len()); + e.rows + .push(((*id).to_owned(), (*label).to_owned(), pcs.to_vec())); + } + e + } + + fn ancient(group: &str) -> AncientRow { + AncientRow { + group: group.to_owned(), + date_bp: None, + country: None, + locality: None, + } + } + + #[test] + fn every_ancient_member_carries_the_distance_pcs() { + // A group whose rows hold 2 PCs must not be ranked on 2 of 4. + let panel = evec( + &[1.0; 4], + &[ + ("a1", ANCIENT_LABEL, &[0.0, 0.0]), + ("a2", ANCIENT_LABEL, &[0.0, 0.0, 9.0, 0.0]), + ], + ); + let ancients = vec![ + ("a1".to_owned(), ancient("Short")), + ("a2".to_owned(), ancient("Full")), + ]; + let Err(Fail(msg)) = nearest_populations(&[0.0, 0.0, 10.0, 0.0], &panel, &ancients, 4) + else { + panic!("a group short of PCs was ranked"); + }; + assert!(msg.contains("ancient group Short has no PC3"), "{msg}"); + } + + #[test] + fn a_person_or_eigenvalue_that_is_not_finite_stops_the_report() { + let d = std::env::temp_dir().join(format!("og-ancient-finite-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let modern: String = (1..=12) + .map(|i| format!("M{i} {}.0 {}.5 {} {} Pop\n", i, 13 - i, i % 3, i % 5)) + .collect(); + let path = |n: &str| d.join(n).to_str().unwrap().to_owned(); + for (person, evals) in [ + ("S1 nan 0.1 0.2 0.3 Person\n", "4 3 2 1"), + ("S1 0.1 0.2 0.3 0.4 Person\n", "4 inf 2 1"), + ] { + std::fs::write( + d.join("panel.evec"), + format!("#eigvals: {evals}\n{modern}A1 1 2 3 4 Ancient\n"), + ) + .unwrap(); + std::fs::write( + d.join("run.evec"), + format!("#eigvals: 4 3 2 1\n{modern}{person}"), + ) + .unwrap(); + std::fs::write(d.join("anc.tsv"), "id\tgroup\nA1\tG\n").unwrap(); + let args: Vec = [ + "report", + "--panel-evec", + &path("panel.evec"), + "--run-evec", + &path("run.evec"), + "--ancients", + &path("anc.tsv"), + "--sample", + "S1", + "--release", + "v1", + "--out-tsv", + &path("o.tsv"), + "--out-info", + &path("o.info"), + "--out-txt", + &path("o.txt"), + "--out-svg", + &path("o.svg"), + ] + .map(str::to_owned) + .to_vec(); + let mut err = Vec::new(); + assert_eq!(cli(&args, &mut Vec::new(), &mut err), 1, "{person}{evals}"); + let err = String::from_utf8(err).unwrap(); + assert!(err.contains("not a finite number"), "{err}"); + assert!(!d.join("o.tsv").exists()); + } + let _ = std::fs::remove_dir_all(&d); + } } diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 4468fc815..6f668b698 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -723,7 +723,9 @@ struct VcfRec { /// The biallelic SNVs (and the other records, as code 9) of a one-sample /// VCF, in file order. A record whose FILTER is not PASS or `.` gives /// code 9 (not read), so a DeepVariant RefCall is neither your allele nor -/// the reference. +/// the reference. Every record is checked against the last record handed +/// out (`previous`), so a record that is skipped cannot hide a chromosome +/// that comes back at a lower position. struct Vcf { text: Text, previous: Option<(String, i64)>, @@ -772,11 +774,11 @@ impl Vcf { self.text.path.display() )); } - self.previous = Some((c[0].to_owned(), pos)); if c[3].chars().count() != 1 || c[4] == "." { continue; } if c[4].contains(',') || c[4].chars().count() != 1 { + self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), pos, @@ -796,6 +798,7 @@ impl Vcf { if c[6] != "PASS" && c[6] != "." { code = 9; } + self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), pos, @@ -922,7 +925,16 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { samples += &format!("{id}\n"); } head.push('\n'); - put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; + // Every output is staged and published only when the whole run has + // passed, so a run that stops leaves the previous set as it was. + let samples_path = Path::new(out_dir).join("samples.txt"); + let mut samples_file = + AtomicFile::create(&samples_path).map_err(|e| cannot_write(&samples_path, &e))?; + samples_file + .write_all(samples.as_bytes()) + .map_err(|e| cannot_write(&samples_path, &e))?; + let mut staged: Vec<(PathBuf, AtomicFile)> = vec![(samples_path, samples_file)]; + let mut stale: Vec = Vec::new(); let vcf_path = a.req("--vcf"); let mut calls = Calls { @@ -942,7 +954,7 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { done.push(chrom.clone()); let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); if let Some(table) = table { - if let Some(span) = write_chrom( + match write_chrom( &chrom, table, &mut calls, @@ -953,7 +965,11 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { two_n, out_dir, )? { - spans.push(span); + Written::Rows(span, files) => { + spans.push(span); + staged.extend(files); + } + Written::Empty(path) => stale.push(path), } } else { while pending.as_ref().is_some_and(|r| r.chrom == chrom) { @@ -961,12 +977,6 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } } } - let mut text = - String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); - for (c, f, l, n, carried, dropped) in &spans { - text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); - } - put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; if spans.is_empty() { let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); let mut sorted = names.clone(); @@ -976,6 +986,25 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { sorted.join(", ") )); } + // Published one rename at a time: the set is not one transaction, and + // a rename that fails part way leaves part of the new set in place. + for (path, file) in staged { + file.commit().map_err(|e| cannot_write(&path, &e))?; + } + for path in stale { + match std::fs::remove_file(&path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(Stop::from(cannot_write(&path, &e))); + } + _ => {} + } + } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; let sites: u64 = spans.iter().map(|s| s.3).sum(); let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); Ok(()) @@ -985,8 +1014,18 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { /// position, sites, sites with your ALT, sites with another allele. type Span = (String, i64, i64, u64, u64, u64); +/// What one chromosome leaves to publish when the whole run has passed. +enum Written { + /// Rows were written: the span row, and the staged genotype file and + /// masks with their final paths. + Rows(Span, Vec<(PathBuf, AtomicFile)>), + /// No row was written: a genotype file of this name from an earlier + /// run is stale and goes. + Empty(PathBuf), +} + /// IBDmix's genotype file of one chromosome, and each archaic genome's -/// mask; the chromosome's span row, or `None` when no row was written. +/// mask, staged under temporary names. #[allow(clippy::too_many_arguments)] fn write_chrom( chrom: &str, @@ -998,7 +1037,7 @@ fn write_chrom( af_col: usize, two_n: f64, out_dir: &str, -) -> Out> { +) -> Out { let num = &chrom[3..]; let path = Path::new(out_dir).join(format!("{chrom}.gt.gz")); let file = AtomicFile::create(&path).map_err(|e| cannot_write(&path, &e))?; @@ -1124,26 +1163,22 @@ fn write_chrom( } if n == 0 { drop(gz); - match std::fs::remove_file(&path) { - Err(e) if e.kind() != io::ErrorKind::NotFound => { - return Err(Stop::from(cannot_write(&path, &e))); - } - _ => {} - } - return Ok(None); + return Ok(Written::Empty(path)); } - gz.finish().and_then(AtomicFile::commit).map_err(werr)?; + let gz = gz.finish().map_err(werr)?; + let mut files = vec![(path, gz)]; // IBDmix's mask (-r): the chromosome as in the genotype file, and a // line covers the positions start + 1 to end, as in a BED. - for (k, mut mask) in masks.into_iter().enumerate() { + for ((k, mut mask), mask_path) in masks.into_iter().enumerate().zip(mask_paths) { if let Some((s, e)) = open_run[k] { - writeln!(mask, "{num}\t{}\t{e}", s - 1) - .map_err(|e| cannot_write(&mask_paths[k], &e))?; + writeln!(mask, "{num}\t{}\t{e}", s - 1).map_err(|e| cannot_write(&mask_path, &e))?; } - mask.commit() - .map_err(|e| cannot_write(&mask_paths[k], &e))?; + files.push((mask_path, mask)); } - Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) + Ok(Written::Rows( + (chrom.to_owned(), first, last, n, carried, dropped), + files, + )) } // --- report (step 43) ------------------------------------------------------------- @@ -1752,6 +1787,126 @@ mod tests { assert!(error.1.contains("not sorted")); } + #[test] + fn an_ignored_record_cannot_hide_a_chromosome_return() { + // chr22 comes back after an ignored chrX record (ALT "."), at a + // lower position: the join would write 0 at chr22:1000 and lose + // the call, so it must stop as unsorted. + let text = "chr22\t3000\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n\ + chrX\t1\t.\tA\t.\t.\tPASS\t.\tGT\t0/0\n\ + chr22\t1000\t.\tC\tT\t.\tPASS\t.\tGT\t1/1\n"; + let mut vcf = Vcf::new(Text { + path: "sample.vcf".into(), + lines: Lines::new( + Box::new(io::Cursor::new(text.as_bytes().to_vec())), + Utf8::Strict, + ), + line: String::new(), + }); + assert_eq!(vcf.next().unwrap().unwrap().pos, 3000); + let error = vcf.next().err().expect("chr22 goes back to 1000"); + assert_eq!(error.0, 2); + assert!( + error.1.contains("chr22:1000 comes after 3000"), + "{}", + error.1 + ); + } + + /// A scratch folder of its own under the system temp folder. + fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-archaic-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("table")).unwrap(); + d + } + + fn gzip_to(path: &Path, text: &str) { + let mut gz = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::new(1)); + gz.write_all(text.as_bytes()).unwrap(); + gz.finish().unwrap(); + } + + /// `og archaic genotypes` on the folder's table into `gt`, with a VCF + /// of these lines: the exit code. + fn genotypes(d: &Path, vcf_lines: &str, pseudo: &str) -> u8 { + let vcf = d.join("s.vcf"); + std::fs::write( + &vcf, + format!("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n{vcf_lines}"), + ) + .unwrap(); + let args: Vec = [ + "genotypes", + "--table-dir", + d.join("table").to_str().unwrap(), + "--vcf", + vcf.to_str().unwrap(), + "--pseudo", + pseudo, + "--out-dir", + d.join("gt").to_str().unwrap(), + ] + .map(str::to_owned) + .to_vec(); + cli(&args, &mut Vec::new(), &mut Vec::new()) + } + + fn snapshot(d: &Path) -> Vec<(String, Vec)> { + let mut files: Vec<(String, Vec)> = std::fs::read_dir(d.join("gt")) + .unwrap() + .map(|e| { + let e = e.unwrap(); + ( + e.file_name().into_string().unwrap(), + std::fs::read(e.path()).unwrap(), + ) + }) + .collect(); + files.sort(); + files + } + + #[test] + fn a_failed_run_leaves_every_previous_output() { + let d = scratch("failed-run"); + let head = TABLE_HEADER.join("\t"); + let row = |c: &str, p: i64, r: &str, a: &str| { + format!("{c}\t{p}\t{r}\t{a}\t0\t1\t12\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n") + }; + gzip_to( + &d.join("table/chr21.tsv.gz"), + &format!( + "{head}\n{}{}", + row("chr21", 100, "A", "G"), + row("chr21", 200, "C", "T") + ), + ); + gzip_to( + &d.join("table/chr22.tsv.gz"), + &format!("{head}\n{}", row("chr22", 100, "G", "A")), + ); + let good = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t100\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n"; + assert_eq!(genotypes(&d, good, "2"), 0); + let before = snapshot(&d); + assert!(before.iter().any(|(n, _)| n == "chr22.gt.gz")); + // chr21 is written, then chr22 is found unsorted: nothing changes, + // not samples.txt for the new --pseudo nor chr21's files. + let unsorted = "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t300\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t400\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t100\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n"; + assert_eq!(genotypes(&d, unsorted, "3"), 2); + assert_eq!(snapshot(&d), before); + // No row written on any chromosome (chr22's only site holds + // another allele): the run fails and chr22's file stays. + let other = "chr22\t100\t.\tG\tC\t.\tPASS\t.\tGT\t1/1\n"; + assert_ne!(genotypes(&d, other, "3"), 0); + assert_eq!(snapshot(&d), before); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn pseudo_columns_carry_the_count() { let p = pseudo_columns(2); diff --git a/tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent b/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb From fa7a6571f57ecb4e8ecc43c93a5b59221e47216a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:41:42 +0100 Subject: [PATCH 19/86] fix(ancestry): remove a stale genotype file only after every new archaic output is in place spans.tsv is now staged with the other outputs, and a chromosome's old genotype file is removed last. A failed publication keeps it. --- rust/crates/og-ancestry/src/archaic.rs | 47 ++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 6f668b698..0dbd7db0c 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -986,8 +986,21 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { sorted.join(", ") )); } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + let spans_path = Path::new(out_dir).join("spans.tsv"); + let mut spans_file = + AtomicFile::create(&spans_path).map_err(|e| cannot_write(&spans_path, &e))?; + spans_file + .write_all(text.as_bytes()) + .map_err(|e| cannot_write(&spans_path, &e))?; + staged.push((spans_path, spans_file)); // Published one rename at a time: the set is not one transaction, and // a rename that fails part way leaves part of the new set in place. + // Stale genotype files go last, once every new file is in place. for (path, file) in staged { file.commit().map_err(|e| cannot_write(&path, &e))?; } @@ -999,12 +1012,6 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { _ => {} } } - let mut text = - String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); - for (c, f, l, n, carried, dropped) in &spans { - text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); - } - put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; let sites: u64 = spans.iter().map(|s| s.3).sum(); let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); Ok(()) @@ -1907,6 +1914,34 @@ mod tests { let _ = std::fs::remove_dir_all(&d); } + #[test] + fn a_failed_publication_keeps_a_stale_genotype_file() { + let d = scratch("failed-publication"); + let head = TABLE_HEADER.join("\t"); + let row = + |c: &str, p: i64| format!("{c}\t{p}\tA\tG\t0\t1\t2\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n"); + gzip_to( + &d.join("table/chr21.tsv.gz"), + &format!("{head}\n{}", row("chr21", 100)), + ); + gzip_to( + &d.join("table/chr22.tsv.gz"), + &format!("{head}\n{}", row("chr22", 100)), + ); + let both = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n"; + assert_eq!(genotypes(&d, both, "2"), 0); + // spans.tsv cannot be replaced: chr22 now has no row, and its old + // genotype file must outlive the failed publication. + std::fs::remove_file(d.join("gt/spans.tsv")).unwrap(); + std::fs::create_dir_all(d.join("gt/spans.tsv/full")).unwrap(); + let chr21_only = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t1/1\nchr22\t100\t.\tA\tC\t.\tPASS\t.\tGT\t1/1\n"; + assert_ne!(genotypes(&d, chr21_only, "2"), 0); + assert!(d.join("gt/chr22.gt.gz").exists()); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn pseudo_columns_carry_the_count() { let p = pseudo_columns(2); From c67f4404e120d5455ded8de7d31f9ba9a098ff2f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:57:41 +0100 Subject: [PATCH 20/86] perf(report): write summary.json with og summary collect render_report.py (step 24, HTML_REPORT and generate-report.sh) now runs og summary collect instead of the Python collector. generate-report.sh renders the text report from the summary step 24 wrote when nothing it reads has changed since, so a run collects each sample once. The folder walk leaves out the Nextflow launch folder run-all.sh keeps in the sample folder. collect_summary.py keeps a collect() that runs og, and only the names other Python still imports. Bead: pgp-bs9u.45 --- bin/collect_summary.py | 4099 +------------------- bin/render_report.py | 12 +- modules/local/html_report/main.nf | 13 +- rust/crates/og-summary/src/collect/scan.rs | 28 +- scripts/24-html-report.sh | 8 +- scripts/generate-report.sh | 47 +- 6 files changed, 235 insertions(+), 3972 deletions(-) diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 054d9a684..a492746d8 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -1,67 +1,127 @@ #!/usr/bin/env python3 -"""collect_summary.py: read one sample's step outputs into one JSON summary. - -Both reports (scripts/24-html-report.sh and scripts/generate-report.sh) are -rendered from this summary by bin/render_report.py, so they cannot disagree -about a count. Standard library only (it runs in the plain python image). - - collect_summary.py --sample S --sample-dir DIR [--out S/summary.json] - -DIR is a bash output folder (GENOME_DIR/S) or a Nextflow one (outdir/S): both -publish names are read (vcf/ or roh/ for ROH, mosdepth/ or coverage/ for -depth, hla_t1k/ or hla/ for HLA, vcf/ or pharmcat/ for PharmCAT). - -Each section has a state: - ok read from its file - missing no file: the step has not run - failed no file, and its step failed in the latest run-all.sh run - (logs/run_status.tsv) - stale the file is older than the latest run-all.sh run and its step was - not ok in that run (logs/run_status.tsv); the values are shown - with the file's date - unreadable the file exists but could not be read (the reason is in note) - -The run manifest (S/run_manifest.tsv, bin/write_manifest.sh) is copied in for -the report footer, with the ClinVar release date and the HLA database release. -Two optional keys, for the report's run facts and badges: recheck, the last -database re-check (S/recheck/state.json, bin/owngenome_recheck.py), and new_since, -the date of the previous run kept in S/history/ and the sections whose results -changed since it (bin/run_delta.py's comparison). - -The step tools that once lived here are og verbs now (rust/crates/og-summary): -`og qc sample` writes step 33's identity and contamination table, `og prs -table` step 25's PRS table, and `og mito annotate|deletions|copies` step 20's -chrM tables. This file reads what they write. +# remnant: kept for the reports, the couple screen, the search index and the live site; deleted in lane R6b +"""collect_summary.py: what other Python still reads of the summary collector. + +The collector itself is `og summary collect` (rust/crates/og-summary): it +reads one sample folder's step outputs into summary.json, the one file both +reports, the live site and the run delta read. collect() below runs it, for +bin/render_report.py (steps 24, HTML_REPORT and generate-report.sh) and for +the live site's fallback while a run has no summary.json yet. The rest are +the names other Python imports until its own port: the section keys, titles +and steps (render_report.py), ClinVar hit rows and the gene tables' allele +cells (couple_report.py), the CSQ rule and the table readers +(results_index.py), and the HLA quality floor (render_report.py). `og dev +remnants` (tests/parity/remnants.tsv) checks that nothing else comes back. +Standard library only. """ -import argparse -import base64 import csv -import glob import gzip import json -import math import os -import re -import struct -import sys +import subprocess +import tempfile import zlib -from datetime import datetime, timezone -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import pgx_outside_calls # noqa: E402 step 36's rule for a CYP2D6 call and an HLA allele -import ancestry_fine_report # noqa: E402 step 26b's group names and painting colours -import paraphase_report # noqa: E402 step 35b's tables and its SMN1 rule beside Parascopy -import clip_check # noqa: E402 CPSR's variant ids and class names, as step 17's read check reads them -import annotsv_postfilter # noqa: E402 step 05's class 4-5 checks and the step 22 support join -import chip_report # noqa: E402 step 49's detection limit at the median depth -import run_delta # noqa: E402 the comparison with the previous run, for the "New since" badges -import traits_report # noqa: E402 step 42's genotype-rules columns and the rules the traits table holds -import genotype_rules_indels # noqa: E402 the APOL1 G2 deletion beside step 42's APOL1 rule +HERE = os.path.dirname(os.path.abspath(__file__)) +# The repository's assets/ folder beside bin/: og summary collect reads the +# ACMG SF rules, the carrier list and the other gene tables there. The +# containers of step 24 and HTML_REPORT mount bin/ alone, so render_report.py +# --assets names the folder instead. +ASSETS_DIR = os.path.join(os.path.dirname(HERE), "assets") + +# The summary's sections in report order: key, title, step. og summary +# collect writes the same keys, titles and steps (rust/crates/og-summary). +SECTIONS = [ + ("coverage", "Coverage (mosdepth)", "16b"), + ("callability", "Gene callability (mosdepth)", "16b"), + ("sex_check", "Sex check (indexcov)", "16"), + ("sample_qc", "Sample identity and contamination (somalier, VerifyBamID2)", "33"), + ("mosaic_sex_loss", "Mosaic loss of chromosome Y or X", "33"), + ("read_count", "Reads kept by alignment", "02"), + ("variants", "Variant calling", "03"), + ("clinvar", "ClinVar screen", "06"), + ("couple", "Couple carrier screen", "39"), + ("carrier", "Carrier list", "06"), + ("pharmcat", "PharmCAT", "07"), + ("cpic", "CPIC drug-gene recommendations", "27"), + ("pypgx", "pypgx", "32"), + ("cyrius", "Cyrius (CYP2D6)", "21"), + ("parascopy", "SMN1/SMN2 copy number (Parascopy, short reads)", "35"), + ("paralogs", "Paralog genes (Paraphase, PacBio HiFi)", "35b"), + ("hla", "HLA typing (T1K)", "08"), + ("prs", "Polygenic risk scores", "25"), + ("ancestry_fine", "Fine-grained ancestry (ADMIXTURE, RFMix)", "26b"), + ("manta", "Structural variants (Manta)", "04"), + ("delly", "Structural variants (Delly)", "19"), + ("cnvpytor", "Copy number (CNVpytor)", "18"), + ("sv_consensus", "SV consensus merge", "22"), + ("annotsv", "Structural variants, class 4 and 5 (AnnotSV)", "05"), + ("expansions", "Repeat expansions (ExpansionHunter)", "09"), + ("telomere", "Telomere content (TelomereHunter)", "10"), + ("roh", "Runs of homozygosity", "11"), + ("haplogroup", "Mitochondrial haplogroup", "12"), + ("y_haplogroup", "Y-chromosome haplogroup (Yleaf)", "37"), + ("y_tree", "Y placement on the current public tree", "37"), + ("y_str", "Y-STR markers (HipSTR)", "45"), + ("consumer_export", "Raw-data export (23andMe v5 layout)", "40"), + ("bloodtype", "Blood groups (predicted)", "41"), + ("lpa", "Lp(a): LPA KIV-2 repeat count", "46"), + ("vntyper", "MUC1 VNTR frameshifts (VNtyper)", "52"), + ("locityper", "Locityper second opinion at hard loci", "53"), + ("mito", "Mitochondrial variants", "20"), + ("traits", "Traits panel", "42"), + ("archaic", "Archaic ancestry (IBDmix)", "43"), + ("ancient", "Ancient populations (AADR PCA)", "44"), + ("viral", "Viral DNA in the unmapped reads", "47"), + ("exomiser", "Symptom-driven ranking (Exomiser)", "48"), + ("chip", "Clonal haematopoiesis (CHIP)", "49"), + ("mei", "Mobile element insertions (MEGAnE)", "50"), + ("mosaic", "Mosaic chromosomal alterations (MoChA)", "51"), + ("cpsr", "Cancer predisposition (CPSR)", "17"), + ("clinical", "Clinical variant filter", "23"), + ("slivar", "Variant prioritization (slivar)", "31"), +] + + +def og(): + """The og binary: OG when set (the tests), else bin/og beside this file.""" + return os.environ.get("OG") or os.path.join(HERE, "og") + + +def collect(sample, sample_dir, declared_sex=None, out=None): + """One sample folder's summary, as og summary collect writes it. With OUT + the summary is also kept in that file; else it goes through a temporary + file, removed after. og's messages go to stderr. RuntimeError when og + writes no summary (the folder cannot be read, or the file cannot be + written).""" + tmp = None + if out is None: + fd, tmp = tempfile.mkstemp(prefix="summary.", suffix=".json") + os.close(fd) + out = tmp + cmd = [og(), "summary", "collect", "--sample", sample, "--sample-dir", sample_dir, + "--assets", ASSETS_DIR, "--out", out] + if declared_sex: + cmd += ["--declared-sex", declared_sex] + try: + try: + # og reads bin/hla_alleles.tsv and bin/str_intermediate_ranges.tsv + # beside this file, wherever the og binary sits. + env = dict(os.environ, OG_BIN_TABLES=os.environ.get("OG_BIN_TABLES", HERE)) + rc = subprocess.run(cmd, stdout=subprocess.DEVNULL, env=env, check=False).returncode + except OSError as e: + raise RuntimeError(f"og summary collect did not start ({cmd[0]}): {e}") from e + if rc != 0: + raise RuntimeError(f"og summary collect wrote no summary of {sample} (exit {rc})") + with open(out, encoding="utf-8") as f: + return json.load(f) + finally: + if tmp: + try: + os.unlink(tmp) + except OSError: + pass -# 2: cyp2d6.consensus, Cyrius's Filter applied, HLA quality per allele, -# Stranger's status on the repeat loci, the CPIC unclassified count, and the -# section state 'failed'. -SCHEMA_VERSION = 2 # ClinVar review status -> stars. og clinvar hits (rust/crates/og-clinvar) holds the same table # (tests/test_collect_summary.py checks that the two agree). @@ -74,20 +134,17 @@ "criteria_provided,_conflicting_interpretations": 1, } + # The call floor of step 6 (scripts/06-clinvar-screen.sh writes a floor other # than this default into the hits VCF as ##PGP_CALL_FLOOR=GQ=..,DP=..): a P/LP # hit below either is a low-confidence call, listed but counted apart. A hits # file without the line used this default, or was written before the floor. CALL_FLOOR = {"GQ": 30, "DP": 10} + + # gnomAD allele frequency above which a P/LP hit is marked common. COMMON_AF = 0.01 -# The genotype check of the clinical filter (bin/csq_rank.awk, step 23 and the -# CLINICAL_FILTER module): step 6's GQ and DP floor, and the alt read fraction -# (AD) a heterozygote must lie within and a homozygous alt call must reach. -# bin/csq_rank.awk writes the floors it used into the clinical VCF as -# ##PGP_CALL_FLOOR=GQ=..,DP=..,HET_AB=lo-hi,HOM_AB=..; these are its defaults. -AB_FLOOR = {"HET_AB": (0.2, 0.8), "HOM_AB": 0.8} -IMPACT_RANK = {"HIGH": 4, "MODERATE": 3, "LOW": 2, "MODIFIER": 1} + # Steps run-all.sh runs and records in logs/run_status.tsv: the rows of # assets/steps.tsv with recorded=yes (tests/test_run_state.py keeps the two @@ -99,50 +156,7 @@ "26b", "27", "28", "29", "30", "31", "32", "33", "35", "35b", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53"} -# What this pipeline does not assess, whatever ran. -NOT_ASSESSED = [ - "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", - "Methylation and imprinting", - "Repeat expansions outside the ExpansionHunter catalog", - "Phase: two variants in one gene are compound-het candidates, not confirmed", - "Variants of uncertain significance (not interpreted)", -] - -# Added to NOT_ASSESSED unless step 35's depth count (og cn call --locus HBA, -# short reads) or step 35b (Paraphase, PacBio HiFi, both genes) made a call. -HBA_NOT_ASSESSED = ("HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count " - "(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call") - -# Added to NOT_ASSESSED when step 50 (opt-in, MEGAnE) did not run; a failed -# step 50 is listed under not run, and unreadable tables under their section. -MEI_NOT_ASSESSED = ("Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): " - "not looked for (step 50, MEGAnE, is opt-in)") - -# Added to NOT_ASSESSED unless step 51 (MoChA) assessed the sample. -MCA_NOT_ASSESSED = ("Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share " - "of the blood cells): step 51 (MoChA) did not assess this sample") - -# Added to NOT_ASSESSED unless step 52 (VNtyper) gave a positive, possible or -# negative reading: no other step can see a frameshift inside the MUC1 VNTR. -VNTYPER_NOT_ASSESSED = ("MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; " - "no standard caller sees them") -# Added to NOT_ASSESSED unless step 25 ran with an ancestry reference panel. -PRS_NOT_ADJUSTED = "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums" - -EH_LOCI = ["HTT", "FMR1", "C9ORF72", "ATXN1", "DMPK"] - -# ACMG SF v3.3 (Lee et al., Genet Med 2025;27(8):101454): the 84 genes in which -# the ACMG recommends reporting pathogenic and likely pathogenic variants found -# by chance, and the reporting rule of each gene (Table 1), are the rows of -# assets/acmg_sf.tsv. A new list version means new rows there and a new -# ACMG_SF_VERSION. The sample's carrier list reads assets/carrier_genes.tsv. -# Both are read from ASSETS_DIR: the repository's assets/ folder beside bin/, -# or the folder render_report.py --assets names (the containers of step 24 and -# of HTML_REPORT mount bin/ only). -ACMG_SF_VERSION = "ACMG SF v3.3" -ASSETS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets") -SF_REPORT_IF = ("any", "biallelic", "hom_variant", "hemizygous", "het_female_too") SEVERITY_KINDS = ("hypomorph", "pseudodeficiency") @@ -150,29 +164,12 @@ class Unreadable(Exception): pass -def iso(ts): - return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - - def open_text(path): if path.endswith((".gz", ".bgz")): return gzip.open(path, "rt", errors="replace") return open(path, errors="replace") -def first_existing(base, rels): - for rel in rels: - for p in sorted(glob.glob(os.path.join(base, rel), recursive=True)): - if os.path.isfile(p): - return p - return None - - -def newest(base, rels): - found = [p for rel in rels for p in glob.glob(os.path.join(base, rel), recursive=True) if os.path.isfile(p)] - return max(found, key=os.path.getmtime) if found else None - - def vcf_records(path): """Yield the split data lines of a VCF (plain or gzip).""" try: @@ -198,27 +195,6 @@ def read_tsv(path): return list(csv.DictReader(f, delimiter="\t")) -# --- sections ------------------------------------------------------------------- - -def sec_variants(d, s): - p = first_existing(d, [f"vcf/{s}.vcf.gz"]) - if not p: - return None, {} - total = passed = snps = indels = 0 - for r in vcf_records(p): - if len(r) < 7: - continue - total += 1 - if r[6] == "PASS": - passed += 1 - alts = [a for a in r[4].split(",") if a not in (".", "*", "")] - if any(len(a) == len(r[3]) == 1 for a in alts): - snps += 1 - if any(len(a) != len(r[3]) and not a.startswith("<") for a in alts): - indels += 1 - return p, {"total": total, "pass": passed, "snps": snps, "indels": indels} - - def clinvar_variants(v, sep): """'chr1:10:C:T|chr1:15:G:A' -> 'chr1:10 C>Tchr1:15 G>A'.""" out = [] @@ -247,22 +223,6 @@ def clinvar_combined_note(info, gt=""): return "." -def call_floor(path): - """Step 6's call floor from its hits VCF header, CALL_FLOOR without the line.""" - floor = dict(CALL_FLOOR) - try: - with open_text(path) as f: - for line in f: - if not line.startswith("##"): - break - if line.startswith("##PGP_CALL_FLOOR="): - for k, v in re.findall(r"(GQ|DP)=(\d+)", line): - floor[k] = int(v) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(path)}: {e}") from e - return floor - - def low_confidence(dp, gq, floor): """True when the call's GQ or DP is known and below the floor.""" return (gq is not None and gq < floor["GQ"]) or (dp is not None and dp < floor["DP"]) @@ -304,753 +264,6 @@ def clinvar_row(r, floor=None): } -def clinvar_cdna_checks(d, s, p=None): - """Step 06's cDNA read check (bin/clip_check.py), keyed by (chrom, pos, - ref, alt), or the table at p. None when the file is absent: the Nextflow - pipeline does not write it.""" - p = p or first_existing(d, [f"clinvar/{s}_clinvar_cdna_check.tsv"]) - if not p: - return None - try: - rows = read_tsv(p) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") from e - - def num(v): - return int(v) if (v or "").isdigit() else None - out = {} - for r in rows: - if not (r.get("pos") or "").isdigit(): - continue - out[(r.get("chrom"), int(r["pos"]), r.get("ref"), r.get("alt"))] = { - "check": r.get("cdna_check") or "not_checked", "note": r.get("cdna_note") or "", - "alt_reads": num(r.get("alt_reads")), "alt_clipped": num(r.get("alt_clipped")), - "ref_reads": num(r.get("ref_reads")), "ref_clipped": num(r.get("ref_clipped")), - "construct": r.get("construct_site") or None} - return out - - -def clinvar_cdna_scan(d, s): - """Step 06's genome-wide cDNA scan (bin/cdna_scan.py): {"note", "loci"}, - or None when the table is absent (a run before the scan existed).""" - p = first_existing(d, [f"clinvar/{s}_clinvar_cdna_scan.tsv"]) - if not p: - return None - try: - with open_text(p) as f: - lines = f.read().splitlines() - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") from e - note = lines[0][len("# cdna_scan: "):] if lines and lines[0].startswith("# cdna_scan: ") else "" - body = [ln for ln in lines if not ln.startswith("#")] - loci = [] - for r in csv.DictReader(body, delimiter="\t"): - if not (r.get("start") or "").isdigit(): - continue - loci.append({"chrom": r.get("chrom"), "start": int(r["start"]), "end": int(r.get("end") or 0), - "junctions": int(r.get("junctions") or 0), "reads": int(r.get("reads") or 0), - "junction_list": r.get("junction_list") or ""}) - return {"note": note, "loci": loci} - - -def clinvar_deep_intronic(d, s): - """Step 06's always-report list (the known deep intronic sites): one row - per site with the sample's call, or None when the table is absent.""" - p = first_existing(d, [f"clinvar/{s}_clinvar_deep_intronic.tsv"]) - if not p: - return None - try: - rows = read_tsv(p) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") from e - return [{"chrom": r.get("chrom"), "pos": int(r["pos"]), "ref": r.get("ref"), "alt": r.get("alt"), - "gene": r.get("gene") or ".", "hgvs": r.get("hgvs") or "", "clinvar_id": r.get("clinvar_id") or "", - "call": r.get("call") or "no call", "detail": r.get("detail") or ""} - for r in rows if (r.get("pos") or "").isdigit()] - - -def tabix_linear_index(path): - """{contig: linear index} of a bgzipped VCF's .tbi: for each 16 kb window, - the virtual file offset of the first record that can overlap it (standard - library only, as the rest of this script).""" - with gzip.open(path + ".tbi", "rb") as f: - b = f.read() - if b[:4] != b"TBI\x01": - raise ValueError(f"{os.path.basename(path)}.tbi is not a tabix index") - n_ref = struct.unpack_from("> 14, len(ioff) - 1)] - if voff == 0xFFFFFFFFFFFFFFFF: - voff = 0 - out, seen = [], False - with open(path, "rb") as raw: - raw.seek(voff >> 16) - g = gzip.GzipFile(fileobj=raw) - g.read(voff & 0xFFFF) - for line in g: - if line.startswith(b"#"): - continue - f = line.split(b"\t", 2) - if len(f) < 3 or not f[1].isdigit(): - continue - if f[0].decode() != chrom: - if seen: - break - continue - seen = True - if int(f[1]) > pos: - break - if int(f[1]) == pos: - out.append(line.decode(errors="replace").rstrip("\n").split("\t")) - return out - - -def vep_gnomad_af(vep, hits): - """gnomAD exome and genome AF and the rsIDs of each hit from step 13's VEP - output, as {(chrom, pos, ref, alt): (exome, genome, rsids)}; a hit with - no record is left out. The fields are VEP's gnomADe_AF, gnomADg_AF and - Existing_variation (--everything); rsids keeps the rs ids of - Existing_variation ("rs123&COSV1" gives ["rs123"]), sorted, [] without - the field.""" - fmt = None - with open_text(vep) as f: - for line in f: - if not line.startswith("#"): - break - if line.startswith("##INFO= 7 else [] - if r[3] != ref or alt not in alts: - continue - # VEP's Allele drops the base an indel's alleles share. - snv = len(ref) == 1 and all(len(a) == 1 for a in alts) - allele = alt if snv or any(a[:1] != ref[:1] for a in alts) else (alt[1:] or "-") - got, ids = [None, None], set() - for tr in info_map(r[7]).get("CSQ", "").split(","): - v = tr.split("|") - if len(v) != len(fmt) or (len(alts) > 1 and v[fmt.index("Allele")] != allele): - continue - for i, c in enumerate(cols): - if c is not None and got[i] is None: - got[i] = num(v[c]) - if ev is not None: - ids.update(x.lower() for x in v[ev].split("&") if re.fullmatch(r"(?i)rs\d+", x)) - out[key] = (got[0], got[1], sorted(ids, key=lambda x: int(x[2:]))) - return out - - -def clinvar_frequencies(d, s, rows): - """Put the gnomAD AF and the rsIDs (rsids, [] when unknown) from step - 13's VEP output on each ClinVar hit, and say where they came from or why - there are none. The hits VCF's ID column is ClinVar's own variation id, - not an rsID, so VEP is where a hit's rsID comes from.""" - vep = first_existing(d, [f"vep/{s}_vep.vcf.gz"]) - note = None - if not vep: - note = "no VEP output (step 13) in this folder" - elif not os.path.exists(vep + ".tbi"): - note = f"{os.path.relpath(vep, d)} has no .tbi index" - found = {} - if note is None: - try: - found = vep_gnomad_af(vep, rows) - except (OSError, EOFError, ValueError, struct.error, gzip.BadGzipFile, zlib.error) as e: - note = f"{os.path.relpath(vep, d)} could not be read: {e}" - for x in rows: - e, g, ids = found.get((x["chrom"], x["pos"], x["ref"], x["alt"]), (None, None, [])) - known = [v for v in (e, g) if v is not None] - x["gnomad_exome_af"], x["gnomad_genome_af"] = e, g - x["rsids"] = ids - x["af"] = max(known) if known else None - x["common"] = x["af"] is not None and x["af"] > COMMON_AF - return {"source": os.path.relpath(vep, d) if note is None else None, "note": note, "common_af": COMMON_AF} - - -def sec_clinvar(d, s): - p = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - if not p: - return None, {} - floor = call_floor(p) - rows = [clinvar_row(r, floor) for r in vcf_records(p) if len(r) >= 8] - rows.sort(key=lambda x: (-x["stars"], x["chrom"], x["pos"])) - # The headline count and the stars leave the low-confidence calls out; - # they stay in the hits, marked, and are counted on their own. - sure = [x for x in rows if not x["low_confidence"]] - by = {str(k): sum(1 for x in sure if x["stars"] == k) for k in (4, 3, 2, 1, 0)} - data = {"count": len(sure), "by_stars": by, "hits": rows, "low_confidence": len(rows) - len(sure), - "call_floor": floor, "frequency": clinvar_frequencies(d, s, rows)} - # The cDNA read check, on each hit when step 06 wrote it. - cdna = clinvar_cdna_checks(d, s) - if cdna is not None: - for x in rows: - x["cdna"] = cdna.get((x["chrom"], x["pos"], x["ref"], x["alt"])) or { - "check": "not_checked", "note": "not in the read-check table", "alt_reads": None, - "alt_clipped": None, "ref_reads": None, "ref_clipped": None} - data["cdna_flagged"] = sum(1 for x in rows if x["cdna"]["check"] == "flagged") - data["construct_sites"] = sum(1 for x in rows if x["cdna"].get("construct")) - scan = clinvar_cdna_scan(d, s) - if scan is not None: - data["cdna_scan"] = scan - deep = clinvar_deep_intronic(d, s) - if deep is not None: - data["deep_intronic"] = deep - return p, data - - -def pharmcat_json(d, s): - return newest(d, [f"pharmcat/{s}.report.json", "pharmcat/*.report.json", - f"vcf/{s}.report.json", "vcf/*.report.json"]) - - -def sec_pharmcat(d, s): - p = pharmcat_json(d, s) - if not p: - return None, {} - try: - with open(p) as f: - data = json.load(f) - except (OSError, ValueError) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") - html = os.path.isfile(p[: -len(".json")] + ".html") - return p, {"version": str(data.get("pharmcatVersion", "")) if isinstance(data, dict) else "", - "html_report": html} - - -def sec_cpic(d, s): - p = first_existing(d, [f"cpic/{s}_phenotypes.tsv"]) - if not p: - return None, {} - genes = [] - for row in read_tsv(p): - genes.append({"gene": row.get("Gene", ""), "diplotype": row.get("Diplotype", ""), - "phenotype": row.get("Phenotype", ""), "status": row.get("Status", "")}) - rec = os.path.join(os.path.dirname(p), f"{s}_cpic_recommendations.txt") - warnings, failed = [], False - if os.path.isfile(rec): - with open(rec, errors="replace") as f: - for line in f: - if "PARSING FAILED" in line: - failed = True - if line.strip().startswith("WARNING:"): - warnings.append(line.strip()[len("WARNING:"):].strip()) - return p, {"genes": genes, - "non_normal": sum(1 for g in genes if g["status"] == "non-normal"), - "ambiguous": sum(1 for g in genes if g["status"] == "ambiguous"), - "not_called": sum(1 for g in genes if g["status"] == "not called"), - "unclassified": sum(1 for g in genes if g["status"] == "unclassified"), - "parse_failed": failed or not genes, - "warnings": warnings} - - -# The rows of the CYP2D6 depth check (bin/cyp2d6_depth_check.py) the reports -# show: its verdict and the MAPQ shares beside it. -DEPTH_CHECK_KEYS = ("status", "gene_mapq1_share", "gene_mapq20_share", "body_mapq1_share", "body_mapq20_share") - - -def sec_pypgx(d, s): - p = first_existing(d, [f"pypgx/{s}_pypgx_summary.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - # A no-call by step 36's list (Indeterminate, FAILED, N/A...), in any case - called = [r for r in rows if (r.get("Diplotype") or "").strip().lower() not in pgx_outside_calls.NO_CALL] - cyp_row = next((r for r in rows if r.get("Gene") == "CYP2D6"), None) - cyp = (cyp_row or {}).get("Diplotype", "") - out = {"genes_total": len(rows), "genes_called": len(called), "cyp2d6": cyp or "not called"} - # pypgx's copy-number call from read depth, one entry per BAM-based gene - # (a VCF gene has none). A gene not callable on this reference gets its - # own line below instead; one without a call ('N/A': pypgx failed, or - # CYP2D6 after a failed depth check) says so, so no gene drops out. - out["cnv_call"] = {} - for r in rows: - status = (r.get("Status") or "").strip() - if (r.get("Source") or "").strip().upper() != "BAM" or status == pgx_outside_calls.NOT_CALLABLE: - continue - v = (r.get("CNV_call") or "").strip() - if v not in ("", ".") and not v.startswith("N/A"): - out["cnv_call"][r["Gene"]] = v - elif status.lower() == "failed" or (r.get("Diplotype") or "").strip().upper() == "FAILED": - out["cnv_call"][r["Gene"]] = "not called: pypgx failed" - elif "depth check" in status or "depth check" in v: - out["cnv_call"][r["Gene"]] = "not called: the CYP2D6 depth check failed" - else: - out["cnv_call"][r["Gene"]] = "not called: pypgx gave no copy-number call" - out["not_callable"] = [r["Gene"] for r in rows - if (r.get("Status") or "").strip() == pgx_outside_calls.NOT_CALLABLE] - out["low_gq"] = [{"gene": r["Gene"], "diplotype": (r.get("Diplotype") or "").strip(), - "min_gq": int(r["Min_GQ"]) if (r.get("Min_GQ") or "").strip().isdigit() else None, - "weakest": (r.get("Weakest_variant") or "").strip()} - for r in rows if (r.get("Quality") or "").strip() == pgx_outside_calls.LOW_GQ] - if cyp_row is not None: - out["cyp2d6_evidence"] = {"phenotype": (cyp_row.get("Phenotype") or "").strip(), - "defining_filter": (cyp_row.get("Defining_filter") or "").strip(), - "quality": (cyp_row.get("Quality") or "").strip()} - dc = os.path.join(os.path.dirname(p), f"{s}_cyp2d6_depth_check.tsv") - if os.path.isfile(dc): - with open(dc, errors="replace") as f: - kv = dict(line.rstrip("\n").split("\t", 1) for line in f if "\t" in line) - out["depth_check"] = {k: kv[k] for k in DEPTH_CHECK_KEYS if k in kv} - comp = os.path.join(os.path.dirname(p), f"{s}_pharmcat_comparison.tsv") - if os.path.isfile(comp): - crow = read_tsv(comp) - out["comparison"] = { - "conflicts": sum(1 for r in crow if r.get("Match") == "No"), - "one_tool_only": sum(1 for r in crow if r.get("Match") in ("pypgx only", "PharmCAT only")), - "concordant": sum(1 for r in crow if r.get("Match") == "Yes"), - } - return p, out - - -def sec_cyrius(d, s): - p = first_existing(d, [f"cyrius/{s}_cyp2d6.tsv", "cyrius/*_cyp2d6.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - r = rows[0] if rows else {} - return p, {"genotype": r.get("Genotype", "") or "none", "filter": r.get("Filter", "")} - - -def paralog_regions(): - """The Paraphase regions the report shows gene by gene, in the order of - assets/paraphase_genes.tsv; the others are counted.""" - out = [] - for r in asset_rows("paraphase_genes.tsv"): - if r["region"] not in out: - out.append(r["region"]) - return out - - -def sec_paralogs(d, s): - """Step 35b (Paraphase, PacBio HiFi): the genes of the regions of - assets/paraphase_genes.tsv, a count of the other regions, the variant rows - per region, and SMN1 beside step 35's Parascopy call when that table is - there too.""" - p = first_existing(d, [f"paralogs/{s}_paraphase_copy_number.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - if rows and not {"region", "gene", "copy_number", "status"} <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} lacks the region, gene, copy_number or status column") - PARALOG_REGIONS = paralog_regions() - genes = [{"region": r["region"], "gene": r["gene"], "copy_number": r["copy_number"], - "status": r["status"], "detail": r.get("detail") or "."} - for r in rows if r["region"] in PARALOG_REGIONS] - genes.sort(key=lambda g: PARALOG_REGIONS.index(g["region"])) - others = [r for r in rows if r["region"] not in PARALOG_REGIONS] - variants = {} - vp = os.path.join(os.path.dirname(p), f"{s}_paraphase_variants.tsv") - if os.path.isfile(vp): - for r in read_tsv(vp): - variants[r.get("region", "")] = variants.get(r.get("region", ""), 0) + 1 - smn1 = next((g["copy_number"] for g in genes if g["gene"] == "SMN1"), None) - smn1 = int(smn1) if smn1 and smn1.isdigit() else None - parascopy = None - pp = os.path.join(os.path.dirname(p), f"{s}_smn_copy_number.tsv") - if os.path.isfile(pp): - parascopy = paraphase_report.parascopy_smn1(pp) - return p, {"genes": genes, - "other_regions": len(others), - "other_regions_called": sum(1 for r in others if r["status"] == "called"), - "variants": variants, - "smn1": {"paraphase": smn1, "parascopy": parascopy, - "agree": paraphase_report.compare_smn1(smn1, parascopy)}} - - -# SMN2 exon 7, c.840 (the base that tells SMN2 from SMN1), GRCh38 1-based; -# SMN1's is paraphase_report.SMN1_EXON7. -SMN2_EXON7 = ("chr5", 70076526) - - -def parascopy_cn_at(rows, chrom, pos): - """The copy number at one position from step 35's table: the - paralog-specific value (psCN) of the row, or of its homologous region, - that holds the position, with the row's aggregate value (agCN). As - paraphase_report.parascopy_smn1: cn is None unless psCN's filter is PASS - and its quality at least 20 (Parascopy's threshold).""" - for r in rows: - try: - regions = [(r["chrom"], int(r["start"]) + 1, int(r["end"]))] - except (KeyError, ValueError, TypeError): # TypeError: a short row (None fields) - continue - for h in (r.get("homologous_regions") or "*").split(","): - m = re.match(r"^([^:]+):(\d+)-(\d+)", h.strip()) - if m: - regions.append((m.group(1), int(m.group(2)), int(m.group(3)))) - idx = next((i for i, (c, b, e) in enumerate(regions) if c == chrom and b <= pos <= e), None) - if idx is None: - continue - cns = (r.get("psCN") or "").split(",") - quals = (r.get("psCN_qual") or "").split(",") - cn_txt = cns[idx] if idx < len(cns) else "?" - q_txt = quals[idx] if idx < len(quals) else "*" - filt = r.get("psCN_filter") or "." - try: - q = float(q_txt) - except ValueError: - q = None - ok = filt == "PASS" and q is not None and q >= 20 and cn_txt.isdigit() - return {"cn": int(cn_txt) if ok else None, - "text": f"{cn_txt} (quality {q_txt}, filter {filt})" + ("" if ok else ", not resolved"), - "agcn": r.get("agCN") or ".", "agcn_qual": r.get("agCN_qual") or ".", - "agcn_filter": r.get("agCN_filter") or "."} - return {"cn": None, "text": "no row covers exon 7", "agcn": ".", "agcn_qual": ".", "agcn_filter": "."} - - -MARKER_FIELDS = ("marker", "rsid", "status", "alt_reads", "depth", "fraction", "copies", "message") - - -def read_marker_table(p, step, key): - """A copy-aware marker table (metric, value; bin/traits_report.py MARKERS): - step 46's KIV-2 4925G>A beside the count or step 35's SMN1 g.27134T>G - beside the psCN. A table that is not there or has no known status reads - as not assessed, with the reason in its message, so the report line - never goes missing beside the count.""" - m = traits_report.MARKERS[key] - what = m["name"] - out = dict.fromkeys(MARKER_FIELDS, ".") - out.update(marker=what, rsid=m["dbsnp"][0], status="not assessed") - if not os.path.isfile(p): - out["message"] = (f"{what}: not assessed, step {step} wrote no {os.path.basename(p)} (a run from before " - "this reader, or the check did not run).") - return out - try: - kv = dict(read_tsv_pairs(p)) - except OSError as e: - kv = {"message": f"{os.path.basename(p)} could not be read: {e}"} - if kv.get("status") not in traits_report.MARKER_STATUSES: - out["message"] = (f"{what}: not assessed, {os.path.basename(p)} has no status row " - f"({', '.join(traits_report.MARKER_STATUSES)}).") - return out - out.update({k: kv[k] for k in MARKER_FIELDS if kv.get(k)}) - return out - - -def smn_reading(smn1): - """What an SMN1 copy number means for SMA, in one line.""" - if smn1 is None: - return "SMN1 not resolved (quality under 20 or filter not PASS): no carrier reading" - if smn1 == 0: - return "no SMN1 copy: the pattern of spinal muscular atrophy (SMA) itself" - if smn1 == 1: - return "one SMN1 copy: the pattern of an SMA carrier" - if smn1 == 2: - return ("two SMN1 copies: not a carrier by copy number; a 2+0 carrier (both copies on one chromosome) " - "cannot be ruled out") - return f"{smn1} SMN1 copies: not a carrier by copy number" - - -# Tables in paralogs/ that other parts of steps 35 and 35b write, not catalog rows. -NOT_CATALOG_TABLES = ("hba", "paraphase") -# agCN_filter and agCN_qual too: without them a catalog locus would read as -# "not resolved" instead of unreadable. -PARASCOPY_COLUMNS = {"chrom", "start", "end", "agCN", "agCN_filter", "agCN_qual", "psCN", "psCN_qual", "psCN_filter"} - - -def tsv_columns(path): - """The header of a tab-separated table as a set ({} for an empty file), so - a table with a header and no row is checked for its columns too.""" - with open_text(path) as f: - return set(next(csv.reader(f, delimiter="\t"), [])) - - -def parascopy_tables(d, s): - """{table: path} of step 35's catalog tables (__copy_number.tsv).""" - out = {} - for p in sorted(glob.glob(os.path.join(glob.escape(os.path.join(d, "paralogs", s)) + "_*_copy_number.tsv"))): - name = os.path.basename(p)[len(s) + 1:-len("_copy_number.tsv")] - if name not in NOT_CATALOG_TABLES and os.path.isfile(p): - out[name] = p - return out - - -def parascopy_locus(name, path, row): - """One catalog table that is not SMN1's: the locus's aggregate copy number - (agCN) per region of its profile, the catalog's genes, tag and reading. - psCN is not read here: the catalog marks it reliable only for SMN1.""" - out = {"table": name, "genes": (row or {}).get("genes") or name, "tag": (row or {}).get("tag") or ".", - "reading": (row or {}).get("reading") or "."} - rows = read_tsv(path) - if not PARASCOPY_COLUMNS <= tsv_columns(path): - return dict(out, status="unreadable", cn=None, - text=f"unreadable: {os.path.basename(path)} lacks the copy-number columns") - na = next((r["agCN_filter"] for r in rows if (r.get("agCN_filter") or "").startswith("not assessed")), None) - if na: - return dict(out, status="not assessed", cn=None, text=na) - regions = [] - for r in rows: - try: - q = float(r.get("agCN_qual") or "") - except ValueError: - q = None - cn = r.get("agCN") or "." - ok = cn.isdigit() and r.get("agCN_filter") == "PASS" and q is not None and q >= 20 - regions.append((r.get("chrom"), r.get("start"), r.get("end"), cn, r.get("agCN_qual") or ".", - r.get("agCN_filter") or ".", ok)) - if not regions: - return dict(out, status="not assessed", cn=None, text="not assessed: the table has no region row") - values = {x[3] for x in regions if x[6]} - if len(values) == 1 and all(x[6] for x in regions): - cn = int(values.pop()) - quals = [float(x[4]) for x in regions] - text = f"{cn} ({len(regions)} region(s), quality {min(quals):g} or more, filter PASS)" - return dict(out, status="called", cn=cn, text=text) - # every region, so a LowQual or discordant one is never hidden; no single - # number, so no catalog reading either (the reading follows a call) - text = "; ".join(f"{c}:{int(b) + 1 if str(b).isdigit() else b}-{e} {cn} (quality {q}, filter {f})" - for c, b, e, cn, q, f, _ in regions) - called = any(x[6] for x in regions) - return dict(out, status="called" if called else "not resolved", cn=None, reading=".", - text=text + ("" if called else ": no region with a whole copy number, filter PASS and quality 20 or more")) - - -NO_SMN_TABLE = "no SMN1 table (step 35 wrote no SMN copy-number table)" - - -def sec_parascopy(d, s): - """Step 35 (Parascopy, short reads): SMN1 and SMN2 copy number at exon 7, - the aggregate copy number of the pair there and the SMA reading; the HBA - count from depth; and every other catalog table, one locus each. Each part - is read on its own, so a catalog without the SMN1 row still shows HBA and - the other loci.""" - tables = parascopy_tables(d, s) - hp = os.path.join(d, "paralogs", f"{s}_hba_copy_number.tsv") - ip = os.path.join(d, "paralogs", f"{s}_cn_intervals.tsv") - cp = os.path.join(d, "paralogs", f"{s}_cyp21a2_psv.tsv") - dp = os.path.join(d, "paralogs", f"{s}_cn_dosage.tsv") - p = (tables.get("smn") or next(iter(tables.values()), None) - or next((x for x in (hp, ip, cp, dp) if os.path.isfile(x)), None)) - if not p: - return None, {} - if "smn" in tables: - rows = read_tsv(tables["smn"]) - if not PARASCOPY_COLUMNS <= tsv_columns(tables["smn"]): - raise Unreadable(f"{os.path.basename(tables['smn'])} lacks the chrom, start, end, agCN, agCN_filter, " - "agCN_qual, psCN, psCN_qual or psCN_filter column") - smn1 = parascopy_cn_at(rows, *paraphase_report.SMN1_EXON7) - smn2 = parascopy_cn_at(rows, *SMN2_EXON7) - # Step 35 writes one row whose agCN_filter is "not assessed: " - # when the locus was not run (no read over it, no model, Parascopy failed). - na = next((r["agCN_filter"] for r in rows if (r.get("agCN_filter") or "").startswith("not assessed")), None) - reading = (f"SMN1 and SMN2 {na}: no carrier reading" if na - else "no row of step 35's table covers SMN1 exon 7: no carrier reading" - if smn1["text"] == "no row covers exon 7" else smn_reading(smn1["cn"])) - else: - rows = [] - smn1 = {"cn": None, "text": NO_SMN_TABLE, "agcn": ".", "agcn_qual": ".", "agcn_filter": "."} - smn2 = {"cn": None, "text": NO_SMN_TABLE} - reading = f"{NO_SMN_TABLE}: no carrier reading" - others = {n: t for n, t in tables.items() if n != "smn"} - catalog = {r["table"]: r for r in asset_rows("paralog_loci.tsv")} if others else {} - return p, {"smn1": smn1, "smn2": {"cn": smn2["cn"], "text": smn2["text"]}, - "agcn": smn1["agcn"], "agcn_qual": smn1["agcn_qual"], "agcn_filter": smn1["agcn_filter"], - # SMN1 g.27134T>G beside the psCN (step 35, _smn1_marker.tsv) - "smn1_marker": read_marker_table(os.path.join(d, "paralogs", f"{s}_smn1_marker.tsv"), "35", - "SMN1_27134"), - "rows": len(rows), - # HBA1/HBA2 from depth (og cn call --locus HBA); None when - # step 35 wrote no HBA table (a run from before the check) - "hba": read_hba(hp) if os.path.isfile(hp) else None, - # the interval rows of bin/cn_loci.tsv (recurrent CNV regions, - # carrier deletions, AZF); None without step 35's table - "cn_intervals": read_cn_intervals(ip) if os.path.isfile(ip) else None, - # CYP21A2 from depth and PSV sites (og cn call, the - # CYP21A2 row): copies, module total, finding; None without it - "cyp21a2": read_hba(cp) if os.path.isfile(cp) else None, - # the gene-dosage rows (HP, LCE3BC, CCL3L1): total copies from - # depth, one entry per locus; None without step 35's table - "dosage": read_cn_dosage(dp) if os.path.isfile(dp) else None, - "reading": reading, - # every other catalog table, in the catalog's order (a table the - # catalog does not name, from a PARALOG_LOCI of the user's, last) - "loci": [parascopy_locus(n, others[n], catalog.get(n)) - for n in sorted(others, key=lambda n: (list(catalog).index(n) if n in catalog - else len(catalog), n))]} - - -def read_cn_intervals(path): - """Step 35's interval table (og cn intervals): the rows with - a deletion or a duplication, and a count of the rows by status.""" - rows = read_tsv(path) - cols = ("locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change") - if not set(cols) <= tsv_columns(path): - raise Unreadable(f"{os.path.basename(path)} lacks the locus, name, status, copies, expected or change column") - short_row(path, rows, cols) - counts = {} - for r in rows: - counts[r["status"]] = counts.get(r["status"], 0) + 1 - return {"rows": len(rows), "by_status": counts, - "changes": [{k: r[k] for k in ("locus", "name", "chrom", "start", "end", "copies", "expected", "change")} - for r in rows if r["change"] in ("deletion", "duplication")]} - - -DOSAGE_COLUMNS = ("locus", "status", "copies", "estimate", "reason", "reading") - - -def read_cn_dosage(path): - """Step 35's gene-dosage table (HP, LCE3BC, CCL3L1): one entry per locus, - in the table's order.""" - if not set(DOSAGE_COLUMNS) <= tsv_columns(path): - raise Unreadable(f"{os.path.basename(path)} lacks the {', '.join(DOSAGE_COLUMNS[:-1])} or reading column") - rows = read_tsv(path) - short_row(path, rows, DOSAGE_COLUMNS) - return [{k: r.get(k) or "." for k in DOSAGE_COLUMNS} for r in rows] - - -def short_row(path, rows, cols): - """A row cut short of a column (csv gives None there) makes the table - unreadable, so it never reads as a row of '.' values.""" - bad = next((i for i, r in enumerate(rows, 2) if any(r.get(k) is None for k in cols)), None) - if bad: - raise Unreadable(f"{os.path.basename(path)} line {bad} lacks a field") - - -def read_hba(path): - """Step 35's HBA table (metric, value) as a dict, and its CYP21A2 table - the same way; a table without a status row reads as not assessed, with - that reason.""" - out = {} - with open(path, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t", 1) - if len(c) == 2 and c[0] != "metric": - out[c[0]] = c[1] - if out.get("status") not in ("called", "not assessed"): - out = {"status": "not assessed", "message": f"not assessed: {os.path.basename(path)} has no status row"} - return out - - -def sec_hla(d, s): - p = first_existing(d, [f"hla_t1k/{s}_hla_genotype.tsv", f"hla/{s}_hla_genotype.tsv", "hla/*_hla_genotype.tsv"]) - if not p: - return None, {} - loci = [] - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 3 or c[0].startswith("#"): - continue - # T1K: allele, abundance, quality, twice; '.', 0, -1 when there is no - # second allele. Each quality stays with its allele. - alleles, quals = [], [] - for i in (2, 5): - a = c[i] if len(c) > i else "" - if a and a != ".": - alleles.append(a) - quals.append(c[i + 2] if len(c) > i + 2 else "") - # Step 36's rule (bin/pgx_outside_calls.py): an allele quality of 0 or - # below withholds the whole gene; only HLA-A and HLA-B reach PharmCAT. - low = any(hla_quality(q) <= 0 for q in quals) - loci.append(hla_locus_extras({"gene": c[0], "alleles": alleles, "quality": quals, "low_confidence": low, - "withheld_from_pharmcat": low and c[0] in pgx_outside_calls.HLA_GENES}, - c[1] if len(c) > 1 else "")) - out = {"loci": loci, "derived": hla_derived(loci)} - ot, kir = hla_optitype(d, s), kir_genotypes(d, s) - if ot is not None: - out["optitype"] = ot - if kir is not None: - out["kir"] = kir - return p, out - - -def first_comment(path): - """The first '#' line of a file step 08 wrote instead of a result, without the '#'.""" - with open(path, errors="replace") as f: - for line in f: - if line.startswith("#"): - return line.lstrip("#").strip() - return "" - - -def hla_optitype(d, s): - """Step 08's second HLA caller (HLA_QC=true): per class I gene the row of - its concordance table (pgx_outside_calls.py hla-concordance), or, when - there is no table, why OptiType's result file says it did not run. None - when OptiType was not asked for.""" - conc = first_existing(d, [f"hla_optitype/{s}_hla_concordance.tsv"]) - res = first_existing(d, [f"hla_optitype/{s}_optitype_result.tsv"]) - if not conc and not res: - return None - if not conc: - return {"genes": [], "reason": first_comment(res) or "no concordance table: step 08 did not compare the callers"} - genes = [] - with open(conc, errors="replace") as f: - for r in csv.DictReader(f, delimiter="\t"): - if (r.get("Gene") or "").strip(): - genes.append({"gene": r["Gene"].strip(), "t1k": (r.get("T1K") or "").strip(), - "optitype": (r.get("OptiType") or "").strip(), "agree": (r.get("Agree") or "").strip(), - "note": (r.get("Note") or "").strip()}) - return {"genes": genes, "reason": "" if genes else "the concordance table has no rows"} - - -def kir_genotypes(d, s): - """Step 08's KIR pass (KIR=true): the genes T1K gave an allele, with each - allele's quality, and the genes it gave none; or the line step 08 wrote - for a BAM with too few KIR reads. None when KIR was not typed.""" - p = first_existing(d, [f"kir_t1k/{s}_kir_genotype.tsv", f"kir/{s}_kir_genotype.tsv"]) - if not p: - return None - typed, not_called = [], [] - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 3 or c[0].startswith("#"): - continue - alleles, quals = [], [] - for i in (2, 5): - if len(c) > i and c[i] not in ("", "."): - alleles.append(c[i]) - quals.append(c[i + 2] if len(c) > i + 2 else "") - if alleles: - typed.append({"gene": c[0], "alleles": alleles, "quality": quals}) - else: - not_called.append(c[0]) - reason = "" if typed or not_called else (first_comment(p) or "the KIR genotype file has no gene rows") - return {"typed": typed, "not_called": not_called, "reason": reason} - - def hla_quality(q): """T1K's quality as a number; one that is not a number counts as -1, as in step 36.""" try: @@ -1059,2000 +272,49 @@ def hla_quality(q): return -1.0 -# --- HLA: candidate caps, the HFE label, the DQ haplotypes and the allele table --- -# Rules in docs/08-hla-typing.md "What the report derives". The allele table is -# bin/hla_alleles.tsv (in bin/ because the report containers mount only bin/). - -HLA_ALLELES_TSV = os.path.join(os.path.dirname(os.path.abspath(__file__)), "hla_alleles.tsv") HLA_MAX_CANDIDATES = 3 # a quality-0 allele with more candidates is "no confident call" -HLA_HFE_LABEL = ("T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the " - "variant calls (ClinVar section)") - - -def hla_name(x): - """'HLA-DQB1*02:01:01' or 'DQB1*02:01:01' -> 'DQB1*02:01:01'; 'HLA-DQB1' -> 'DQB1'.""" - return x.strip()[4:] if x.strip().startswith("HLA-") else x.strip() - - -def hla_candidates(allele): - """T1K's allele field: one allele, or equally likely candidates separated by commas.""" - return [a for a in (hla_name(x) for x in allele.split(",")) if a] - - -def hla_locus_extras(locus, n_diff): - """Cap a quality-0 candidate list to HLA_MAX_CANDIDATES (the full list stays - in the TSV), count the candidates, mark one homozygous call and label the - HFE row.""" - counts, kept = [], [] - for a, q in zip(locus["alleles"], locus["quality"] + [""] * len(locus["alleles"])): - cands = hla_candidates(a) - counts.append(len(cands)) - kept.append(",".join(cands[:HLA_MAX_CANDIDATES]) if hla_quality(q) <= 0 and len(cands) > HLA_MAX_CANDIDATES - else a) - locus["alleles"] = kept - locus["candidates"] = counts - # T1K writes one allele and '. 0 -1' for a homozygous gene (or a second allele it did not see). - locus["one_allele_type"] = n_diff.strip() == "1" and len(kept) == 1 - if hla_name(locus["gene"]) == "HFE": - locus["label"] = HLA_HFE_LABEL - return locus - - -def hla_match(allele, group): - """'yes' when every candidate of the allele is in the group (B*57:01 holds - B*57:01:01:02), 'no' when none is, 'mixed' when only some are.""" - hits = [c == group or c.startswith(group + ":") for c in hla_candidates(allele)] - return "yes" if hits and all(hits) else ("mixed" if any(hits) else "no") - - -def hla_read_table(path=HLA_ALLELES_TSV): - """bin/hla_alleles.tsv as a list of dicts (tests/test_hla_report.py checks its schema).""" - rows, head = [], None - with open(path) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if head is None: - head = c - continue - rows.append(dict(zip(head, c))) - return rows - - -def hla_derived(loci, table=None): - """The DQ haplotypes, the DQB1*02 dose and each table allele present or absent.""" - by = {} - for l in loci: - by.setdefault(hla_name(l["gene"]), l) - - def usable(gene): - l = by.get(gene) - if not l or not l["alleles"]: - return None, f"HLA-{gene} not typed" - if l["low_confidence"]: - return None, f"HLA-{gene} low confidence" - return l, "" - - def carries(l, group): - """'yes', 'no' or 'mixed' over both alleles of a locus.""" - m = [hla_match(a, group) for a in l["alleles"]] - return "yes" if "yes" in m else ("mixed" if "mixed" in m else "no") - - def both(a, b): - return "present" if a == b == "yes" else ("absent" if "no" in (a, b) else "uncertain") - - dqa, ra = usable("DQA1") - dqb, rb = usable("DQB1") - coeliac = {"status": "not_determined", "reason": "; ".join(r for r in (ra, rb) if r), - "dq2_5": None, "dq2_5_phase": None, "dq2_2": None, "dq8": None, "dqb1_02_dose": None} - if dqb: - m = [hla_match(a, "DQB1*02") for a in dqb["alleles"]] - if "mixed" not in m: - n = m.count("yes") - coeliac["dqb1_02_dose"] = 2 * n if dqb["one_allele_type"] else n - if dqa and dqb: - coeliac.update(status="derived", reason="") - coeliac["dq2_5"] = both(carries(dqa, "DQA1*05"), carries(dqb, "DQB1*02")) - coeliac["dq2_2"] = both(carries(dqa, "DQA1*02:01"), carries(dqb, "DQB1*02:02")) - coeliac["dq8"] = both(carries(dqa, "DQA1*03"), carries(dqb, "DQB1*03:02")) - if coeliac["dq2_5"] == "present": - drb1, why = usable("DRB1") - dr3 = carries(drb1, "DRB1*03") if drb1 else None - coeliac["dq2_5_phase"] = ("likely cis (DRB1*03 present)" if dr3 == "yes" - else "possible trans (no DRB1*03)" if dr3 == "no" - else f"phase unknown ({why or 'DRB1*03 uncertain'})") - alleles = [] - for r in (hla_read_table() if table is None else table): - gene = r["allele"].split("*")[0] - l, why = usable(gene) - state = carries(l, r["allele"]) if l else None - alleles.append({"allele": r["allele"], "kind": r["kind"], "association": r["association"], - "pmids": r["pmids"], "citation": r["citation"], - "state": {"yes": "present", "no": "absent", "mixed": "uncertain", None: "not_determined"}[state], - "reason": why}) - return {"coeliac": coeliac, "alleles": alleles} - - -def sec_prs(d, s): - p = first_existing(d, [f"prs/{s}_prs_summary.tsv"]) - if not p: - return None, {} - rows = [] - for r in read_tsv(p): - pct = r.get("Percentile", "") or "" - rows.append({"condition": r.get("Condition", ""), "pgs_id": r.get("PGS_ID", ""), - "score": r.get("Score_SUM", ""), "matched": r.get("Variants_Matched", ""), - "total": r.get("Variants_Total", ""), - "chrx": r.get("ChrX", ""), - # "12 dropped", "12 kept", "12 dropped, strand check failed", "0"; empty in older tables - "ambiguous": r.get("Ambiguous", ""), - # the sex a sex-specific score applies to, and the check against the declared sex - "applies_to": r.get("Applies_To", ""), "sex_check": r.get("Sex_Check", ""), - "percentile": "" if pct == "NA" else pct, - "group": "" if r.get("Ancestry_Group", "NA") == "NA" else r.get("Ancestry_Group", ""), - # the groups the PGS Catalog has an OR or HR of the score for, and the - # one of the sample's group ("HR 1.53 (EUR, PPM000591)"); empty in older tables - # None when the table has no Evaluated_In column (a score list without - # evaluations, or an older table): nothing is known, so nothing is said - "evaluated_in": ([x for x in r["Evaluated_In"].split(";") if x] - if r.get("Evaluated_In") is not None else None), - "effect": r.get("Effect", "") or ""}) - data = {"scores": rows, - # Percentiles exist only when step 25 ran pgsc_calc with an ancestry panel. - "adjusted": any(r["percentile"] for r in rows)} - a = first_existing(d, [f"ancestry/{s}_ancestry.tsv", f"prs/{s}_ancestry.tsv"]) - if a: - kv = {r.get("key"): r.get("value", "") for r in read_tsv(a) if r.get("key")} - data["ancestry"] = {"population": kv.get("population", ""), "panel": kv.get("reference_panel", ""), - "low_confidence": kv.get("population_low_confidence", "")} - return p, data - - -AUTOSOMES = frozenset(str(i) for i in range(1, 23)) - - -def count_vcf(p): - total = passed = 0 - for r in vcf_records(p): - total += 1 - if len(r) > 6 and r[6] == "PASS": - passed += 1 - return total, passed - - -def sec_ancestry_fine(d, s): - """Step 26b: _ancestry_fine.tsv (level, code, group, fraction, se, low, - high; levels continent, region and painting, plus left_out for a group the - model leaves out and not_painted for a chromosome RFMix failed on) and - the painting picture.""" - p = first_existing(d, [f"ancestry_fine/{s}_ancestry_fine.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = {"level", "code", "group", "fraction", "se", "low", "high"} - if not rows or not need <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {sorted(need)}") - out = {"continent": [], "region": [], "painting": [], "left_out": [], "not_painted": [], - "min_references": ancestry_fine_report.MIN_REFERENCES} - for r in rows: - if r["level"] in out: - out[r["level"]].append({"code": r["code"], "group": r["group"], "fraction": num(r["fraction"]), - "low": num(r["low"]), "high": num(r["high"])}) - # The painting picture travels in the summary, so the HTML report shows it - # from the summary alone (render_report.py --summary). - png = first_existing(d, [f"ancestry_fine/{s}_painting.png"]) - out["painting_png"] = None - out["legend"] = [{"code": c, "group": ancestry_fine_report.label("continent", c), - "colour": "#%02x%02x%02x" % rgb} for c, rgb in ancestry_fine_report.COLOURS.items()] - if png and os.path.getsize(png) < 2_000_000: - with open(png, "rb") as f: - out["painting_png"] = base64.b64encode(f.read()).decode() - return p, out - - -def sec_ancient(d, s): - """Step 44: ancient/_ancient.tsv (the nearest ancient groups on the AADR - panel's PCs), _ancient_info.tsv (release, sites used, people, the - sample's PC1 and PC2) and the plot, which travels in the summary so the - HTML report shows it from the summary alone.""" - p = first_existing(d, [f"ancient/{s}_ancient.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = {"rank", "group", "n", "distance", "dates", "country", "locality"} - if not rows or not need <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {sorted(need)}") - info = {} - ip = first_existing(d, [f"ancient/{s}_ancient_info.tsv"]) - if ip: - info = {r.get("key"): r.get("value") for r in read_tsv(ip)} - out = {"nearest": [{k: r[k] for k in ("rank", "group", "n", "distance", "dates", "country", "locality")} for r in rows], - "release": info.get("release") or ".", "sites_used": info.get("sites_used") or ".", - "panel_sites": info.get("panel_sites") or ".", "ancient_people": info.get("ancient_people") or ".", - "ancient_groups": info.get("ancient_groups") or ".", "present_day_people": info.get("present_day_people") or ".", - "plot_svg": None} - svg = first_existing(d, [f"ancient/{s}_ancient_pca.svg"]) - if svg and os.path.getsize(svg) < 3_000_000: - with open(svg, "rb") as f: - out["plot_svg"] = base64.b64encode(f.read()).decode() - return p, out - - -def sec_manta(d, s): - p = first_existing(d, ["manta/results/variants/diploidSV.vcf.gz", "manta/*diploidSV.vcf.gz", - "manta/**/diploidSV.vcf.gz"]) - if not p: - return None, {} - t, ps = count_vcf(p) - return p, {"total": t, "pass": ps} - - -def sec_delly(d, s): - p = first_existing(d, [f"delly/{s}_sv.vcf.gz"]) - if not p: - return None, {} - t, ps = count_vcf(p) - return p, {"total": t, "pass": ps} - - -# A CNVpytor call is kept in the filtered count when fewer than half of its -# reads have mapping quality 0 (q0, column 9) and fewer than half of its bases -# are N (pN, column 10), it touches no centromere or heterochromatin band -# (MASKED_BANDS), and it is not on chrY of a declared female (collect() takes -# those out, once the declared sex is known). -CNV_MAX_Q0 = CNV_MAX_PN = 0.5 - - -def sec_cnvpytor(d, s): - p = first_existing(d, [f"cnvpytor/{s}_cnvs.txt"]) - if not p: - return None, {} - bands = masked_bands() - total = dels = dups = sig = kept = kept_y = 0 - dropped = {"q0": 0, "gap": 0, "band": 0} - with open(p, errors="replace") as f: - for line in f: - c = line.split() - if not c: - continue - total += 1 - dels += c[0] == "deletion" - dups += c[0] == "duplication" - m = re.fullmatch(r"(\S+):(\d+)-(\d+)", c[1]) if len(c) > 4 else None - try: - if float(c[4]) >= 0.01: - continue - except (IndexError, ValueError): - continue - sig += 1 - q0, pn = (num(c[8]), num(c[9])) if len(c) > 9 else (None, None) - if q0 is not None and q0 >= CNV_MAX_Q0: - dropped["q0"] += 1 - elif pn is not None and pn >= CNV_MAX_PN: - dropped["gap"] += 1 - elif m and masked_overlap(bands, m.group(1), int(m.group(2)), int(m.group(3))): - dropped["band"] += 1 - else: - kept += 1 - kept_y += bool(m) and m.group(1) in ("chrY", "Y") - return p, {"total": total, "deletions": dels, "duplications": dups, "significant": sig, - "significant_filtered": kept, "filtered_chry": kept_y, "dropped": dropped} - - -def cnvpytor_sex_filter(data, declared_sex): - """Take the chrY calls out of the filtered count of a declared female; - with no declared sex they stay, and the data says so.""" - if "significant_filtered" not in data: - return - data["declared_sex"] = declared_sex or None - if declared_sex == "female": - data["dropped"]["chry_female"] = data["filtered_chry"] - data["significant_filtered"] -= data["filtered_chry"] - -def sec_sv_consensus(d, s): - p = first_existing(d, [f"sv_merged/{s}_sv_consensus.vcf.gz"]) - if not p: - return None, {} - t, _ = count_vcf(p) - return p, {"consensus": t} - -def sec_annotsv(d, s): - """Step 05's class 4 and 5 SVs (annotsv/_sv_class4_5_checked.tsv, - bin/annotsv_postfilter.py), each with how many callers support it from - step 22's consensus (sv_merged/_sv_consensus.vcf.gz), joined here - because step 22 runs after step 05 (and beside ANNOTSV in Nextflow).""" - p = first_existing(d, [f"annotsv/{s}_sv_class4_5_checked.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) if os.path.getsize(p) else [] - need = {"SV_chrom", "SV_start", "SV_end", "SV_type", "ACMG_class", "Caller", "Exon_edge", "Population_AF", "Common"} - if rows and not need <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {sorted(need)}") - cons = first_existing(d, [f"sv_merged/{s}_sv_consensus.vcf.gz"]) - cons_note = None - try: - support = annotsv_postfilter.consensus_support(rows, cons) - except (OSError, EOFError, zlib.error, UnicodeDecodeError) as e: - support = annotsv_postfilter.consensus_support(rows, None) - cons_note = f"{os.path.basename(cons)} could not be read: {e}" - cons = None - out = [] - for r, sp in zip(rows, support): - try: - af = float(r["Population_AF"]) if r["Population_AF"] else None - except ValueError: - af = None - genes = [g for g in (r.get("Gene_name") or "").split(";") if g] - fmt = annotsv_postfilter.sample_format(r) - out.append({"sv": f"{r['SV_chrom']}:{r['SV_start']}-{r['SV_end']}", "type": r["SV_type"], - # the call's genotype and phase set (PS), for the two-hit rule (two_hits) - "gt": fmt.get("GT") or None, "ps": fmt.get("PS") or None, - "length": r.get("SV_length") or "", "acmg_class": r["ACMG_class"], "genes": genes, - "caller": r["Caller"], "support": sp["Support"], - "support_callers": sp["Support_callers"].split(",") if sp["Support_callers"] else [], - "exon_edge": r["Exon_edge"], "exon_edge_detail": r.get("Exon_edge_detail") or "", - "population_af": af, "population_source": r.get("Population_source") or "", - "common": r["Common"] == "yes", "annotsv_class": r.get("AnnotSV_class") or r["ACMG_class"], - "rule": r.get("Rule") or "", "rule_detail": r.get("Rule_detail") or ""}) - # A call step 05's rules moved from AnnotSV's class 4 or 5 to class 3 - # (bin/annotsv_postfilter.py: a coding tandem repeat, an IG/TR locus, the - # MHC, a pseudogene alias, no read-depth support) is listed apart with - # the reason, and the class 4-5 counts leave it out. - moved = [x for x in out if str(x["acmg_class"]) not in ("4", "5")] - out = [x for x in out if str(x["acmg_class"]) in ("4", "5")] - return p, {"count": len(out), "rows": out, "downgraded": moved, - "consensus": os.path.relpath(cons, d) if cons else None, "consensus_note": cons_note, - "single_caller": sum(x["support"] == 1 for x in out), - "exon_edge": sum(x["exon_edge"] == "yes" for x in out), - "exon_edge_checked": not any(x["exon_edge"] == "not checked" for x in out), - "common": sum(x["common"] for x in out)} - - -STR_RANGES_TSV = os.path.join(os.path.dirname(os.path.abspath(__file__)), "str_intermediate_ranges.tsv") -# How well ExpansionHunter sized an allele, from its FORMAT SO, ADSP and -# REPCI fields, worst first; a locus takes the worst of its alleles. -STR_SIZING = ["no_call", "flanking_only", "in_repeat", "low_confidence", "sized", "unknown"] -STR_MIN_SPANNING = 2 - - -def str_ranges(path=STR_RANGES_TSV): - """bin/str_intermediate_ranges.tsv by locus (tests/test_str_ranges.py checks it).""" - rows, head = {}, None - with open(path) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if head is None: - head = c - continue - r = dict(zip(head, c)) - rows[r["locus"]] = r - return rows - - -def str_int(x): - try: - return int(x) - except (TypeError, ValueError): - return None - - -def str_allele_sizing(so, adsp, ci, cn, path_min): - """One allele's sizing: sized, low_confidence (fewer than - STR_MIN_SPANNING spanning reads, or a confidence interval that reaches - the catalog's pathogenic minimum from below it), in_repeat (in-repeat - reads only: longer than a read), flanking_only, no_call, or unknown - (no SO field).""" - if str_int(cn) is None: - return "no_call" - if not so or so == ".": - return "unknown" - if so == "FLANKING": - return "flanking_only" - if so == "INREPEAT": - return "in_repeat" - hi = str_int(ci.split("-")[-1]) if ci else None - if (str_int(adsp) is not None and str_int(adsp) < STR_MIN_SPANNING) or ( - path_min is not None and hi is not None and str_int(cn) < path_min <= hi): - return "low_confidence" - return "sized" - - -def str_field(fmt, val, key): - return val[fmt.index(key)] if key in fmt and fmt.index(key) < len(val) else "" - - -def str_split(x, n): - """A per-allele FORMAT value as n parts; one value serves every allele.""" - parts = x.split("/") if x else [] - return parts if len(parts) == n else (parts * n if len(parts) == 1 else [""] * n) - - -def rfc1_motif_row(d, s, after): - """Step 9b's RFC1 motif count (bin/rfc1_motif.py), unless it is older than - the Stranger file it was made from.""" - p = first_existing(d, [f"expansion_hunter/{s}_rfc1_motif.tsv"]) - if not p or (after and os.path.getmtime(p) < os.path.getmtime(after)): - return None - rows = read_tsv(p) - if not rows: - return None - r = rows[0] - return {k: r.get(k, ".") for k in ("verdict", "note", "dominant_motif", "dominant_class", - "dominant_share", "motif_shares", "repeat_reads")} - - -def str_motif_rows(d, s, after): - """Step 9b's motif count of the other flagged motif loci (the FAME loci, - SCA31, SCA37; expansion_hunter/_str_motifs.tsv, bin/rfc1_motif.py - --flagged) by locus, unless the table is older than the Stranger file it - was made from.""" - p = first_existing(d, [f"expansion_hunter/{s}_str_motifs.tsv"]) - if not p or (after and os.path.getmtime(p) < os.path.getmtime(after)): - return {} - return {r["locus"]: {k: r.get(k, ".") for k in ("verdict", "note", "dominant_motif", "dominant_class", - "dominant_share", "motif_shares", "repeat_reads")} - for r in read_tsv(p) if r.get("locus")} - - -def str_not_graded(d, s): - """{locus: reason} of the loci the catalog ExpansionHunter ran on gives no - thresholds (its NoThresholds field, og str catalog: MUC1, CEL, - TYMS and PLIN4 among them), from the copy step 09 keeps beside its VCF - (expansion_hunter/_eh_catalog.json). Empty without the copy, or with - one that is not a catalog (Stranger's bundled catalog has no such loci).""" - p = first_existing(d, [f"expansion_hunter/{s}_eh_catalog.json"]) - if not p: - return {} - try: - with open(p, encoding="utf-8") as f: - cat = json.load(f) - except (OSError, ValueError): - return {} - if not isinstance(cat, list): - return {} - return {e["LocusId"]: str(e["NoThresholds"]) for e in cat - if isinstance(e, dict) and e.get("LocusId") and e.get("NoThresholds")} - - -# The repeats STRchive leaves out (assets/str_extra_loci.json), read from -# their counts when step 09 sized them (STR_EXTRA_LOCI=yes; docs/09-str-expansions.md). -STR_EXTRA_SETUP = "STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data " -# What a section state other than ok says about a table another step wrote. -OTHER_STEP_STATE = {"stale": "from an earlier run", "failed": "its step failed in the last run", - "unreadable": "unreadable"} -# CFTR intron 9 (legacy IVS8): ExpansionHunter writes the TG and the poly-T -# tract as two records, so which TG count sits on the copy of which T count -# is known only when one of the two is the same on both copies. R117H -# (rs78655421, chr7:117530975 G>A) is step 42's traits row, 17.6 kb from the -# tract, farther than a read pair spans: with both heterozygous its phase is -# not known. Deignan et al. 2020 (PMID 32404922). -CFTR_R117H = "rs78655421" -CFTR_SOURCE = "Deignan et al. 2020, PMID 32404922" -# UGT1A1 promoter: ExpansionHunter counts the TA units from the first T of -# A(TA)6TAA, so its count is one more than the allele's TA number. -UGT1A1_ALLELES = {6: "*36", 7: "*1", 8: "*28", 9: "*37"} -UGT1A1_DECREASED = ("*28", "*37") -UGT1A1_SOURCE = "Gammal et al. 2016, PMID 26417955" -HTTLPR_READING = ("not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found " - "that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would " - "carry no recommendation.") - - -def str_counts(item): - """The two repeat counts of a sized record, sorted, or None.""" - if not item or item["sizing"] not in ("sized", "unknown"): - return None - c = [str_int(x) for x in item["repeat_count"].split("/")] - if len(c) == 1 and c[0] is not None: - c = c * 2 # a one-copy call (a haploid contig): not expected on chr7 or chr2 - return sorted(c) if len(c) == 2 and None not in c else None +# Pages that show more of a called branch, free and with no upload. The URL +# patterns were checked live on 2026-10-25 (docs/37-y-haplogroup.md, +# "Explore your branch further"): FamilyTreeDNA Discover answers its own +# HaplogroupSearch for R-M269 and H5a7, and YFull's tree pages answer 200 for +# a branch it has and 404 for one it does not. +EXPLORE_DOCS = "https://github.com/GeiserX/OwnGenome/blob/main/docs/" -def trait_genotype(d, s, rsid): - """(status, genotype) of a traits table row of step 42 (traits/_traits.tsv), or None.""" - p = first_existing(d, [f"traits/{s}_traits.tsv"]) - if not p: - return None - try: - row = next((r for r in read_tsv(p) if r.get("rsid") == rsid), None) - except (OSError, csv.Error): - return None - return (row.get("status") or "", row.get("genotype") or "") if row else None +def csq_rank(fmt, csq): + """The rule of bin/csq_rank.awk on one CSQ value: (class, entries, freq). + class is MANE (entries of the MANE Select transcript), else CANONICAL, + else ANY (every entry); freq is rare, common, unknown (no value) or none + (the CSQ has no frequency field): MAX_AF, else gnomADe_AF and gnomADg_AF, + the highest over the entries.""" + ix = {k: i for i, k in enumerate(fmt)} + entries = [v for v in (t.split("|") for t in csq.split(",") if t) if len(v) == len(fmt)] + def get(v, k): + return v[ix[k]] if k in ix else "" + mane = [v for v in entries if get(v, "MANE_SELECT")] + canon = [v for v in entries if get(v, "CANONICAL") == "YES"] + cls, chosen = ("MANE", mane) if mane else ("CANONICAL", canon) if canon else ("ANY", entries) + fields = ["MAX_AF"] if "MAX_AF" in ix else [f for f in ("gnomADe_AF", "gnomADg_AF") if f in ix] + afs = [] + for v in entries: + for f in fields: + for a in get(v, f).split("&"): + try: + afs.append(float(a)) + except ValueError: + pass + freq = "none" if not fields else "unknown" if not afs else "rare" if max(afs) < COMMON_AF else "common" + return cls, [{"impact": get(v, "IMPACT"), "gene": get(v, "SYMBOL"), + "consequence": get(v, "Consequence").replace("&", ",")} for v in chosen], freq -def r117h_genotype(d, s, state): - """(genotype or None, why not read, note) of R117H in step 42's traits - table, given the traits section's state: a table of a failed or - unreadable step is not read, one from an earlier run is read and said so.""" - if state in ("failed", "unreadable"): - return None, f"step 42's traits table is {OTHER_STEP_STATE[state]}", "" - r = trait_genotype(d, s, CFTR_R117H) - if not r or r[0] != "called": - return None, "no step 42 traits row for rs78655421 was called", "" - return r[1], "", (" (R117H from step 42's traits table of an earlier run)" if state == "stale" else "") - -def cftr_reading(by, d, s, traits_state="ok"): - tg, t = by.get("CFTR_TG"), by.get("CFTR_T") - out = {"locus": "CFTR", "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", "source": CFTR_SOURCE} - if not tg or not t: - return dict(out, status="not_sized", genotype=".", phase=".", - reading=f"not in step 09's output: {STR_EXTRA_SETUP} adds the tract to the catalog " - "(docs/09-str-expansions.md).") - a, b = str_counts(tg), str_counts(t) - if not a or not b: - return dict(out, status="not_sized", genotype=".", phase=".", - reading=f"ExpansionHunter did not size both tracts (TG {tg['repeat_count']}, {tg['sizing']}; " - f"poly-T {t['repeat_count']}, {t['sizing']}): not read.") - if a[0] == a[1] or b[0] == b[1]: - pairs = [f"TG{a[0]}-{b[0]}T/TG{a[1]}-{b[1]}T"] - else: - pairs = [f"TG{a[0]}-{b[0]}T/TG{a[1]}-{b[1]}T", f"TG{a[1]}-{b[0]}T/TG{a[0]}-{b[1]}T"] - genotype = pairs[0] if len(pairs) == 1 else f"TG{a[0]}/TG{a[1]} {b[0]}T/{b[1]}T" - out.update(status="read", genotype=genotype, pairings=pairs, tg=a, poly_t=b) - five = b.count(5) - other = sorted({n for n in b if n not in (5, 7, 9)}) - odd = (f" A poly-T length of {', '.join(f'{n}T' for n in other)} is none of the usual 5T, 7T and 9T." - if other else "") - r117, why, note = r117h_genotype(d, s, traits_state) - if not five: - # No 5T risk of its own, but the poly-T still sets how R117H reads: - # with no 5T copy, an R117H copy sits on a 7T or 9T (or other) copy. - line = (f"No 5T copy ({b[0]}T/{b[1]}T): the tract adds no 5T allele of its own. The poly-T length still " - "matters beside R117H, which is milder and of variable penetrance on a 7T or 9T copy and CF-causing " - "with 5T in cis.") - if r117 is None: - line += f" R117H was not read ({why})." - elif r117.split("/").count("A") == 2: - copies = f"both {b[0]}T copies" if b[0] == b[1] else f"both copies ({b[0]}T and {b[1]}T)" - line += f" R117H ({r117}) is read: it sits on {copies}.{note}" - elif r117.split("/").count("A"): - copies = f"{b[0]}T" if b[0] == b[1] else f"{b[0]}T or {b[1]}T" - line += f" R117H ({r117}) is read: with no 5T copy it sits on a {copies} copy.{note}" - else: - line += f" No R117H copy.{note}" - return dict(out, five_t=False, phase="not_needed", r117h=r117 or ".", reading=line + odd) - if len(pairs) == 1: - # One TG count on both copies, or 5T on both copies with their two TG counts. - tgs = [a[0]] if a[0] == a[1] else a - tg_text = " and ".join(f"TG{x}" for x in tgs) + (" on the 5T copy" if five == 1 else " on the 5T copies") - else: - tg_text = f"TG{a[0]} or TG{a[1]} on the 5T copy, not told by the counts ({' or '.join(pairs)})" - line = (f"5T: {five} copy{'s' if five > 1 else ''} with the 5T poly-T tract, {tg_text}. 5T lowers the splicing " - "of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often " - "causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant.") - if r117 is None: - phase = "not_read" - text = f" R117H was not read ({why}), so whether it sits on the 5T copy is not known." - else: - n = r117.split("/").count("A") - if n == 0: - phase = "not_needed" - text = " No R117H copy: the 5T is read on its own." - elif n == 2 or five == 2: - phase = "in_cis" - text = (f" R117H ({r117}) and 5T share a copy: R117H with 5T in cis is a CF-causing allele of " - "variable penetrance.") - else: - phase = "unknown" - text = (" One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther " - "than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable " - "penetrance) or not (in trans) is not read; testing a parent tells them apart.") - text += note - return dict(out, five_t=True, phase=phase, r117h=r117 or ".", reading=line + text + odd) - - -def ugt1a1_reading(by, d, s, cpic_state="ok"): - item = by.get("UGT1A1") - out = {"locus": "UGT1A1", "repeat": "promoter TA repeat", "source": UGT1A1_SOURCE} - if not item: - return dict(out, status="not_sized", genotype=".", pharmcat=".", agreement="not_compared", - reading=f"not in step 09's output: {STR_EXTRA_SETUP} adds the repeat to the catalog " - "(docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28.") - c = str_counts(item) - if not c: - return dict(out, status="not_sized", genotype=".", pharmcat=".", agreement="not_compared", - reading=f"ExpansionHunter did not size the repeat ({item['repeat_count']}, {item['sizing']}): " - "not read.") - names = [UGT1A1_ALLELES.get(n, f"TA{n - 1}") for n in c] - genotype = "/".join(names) - dec = sum(n in UGT1A1_DECREASED for n in names) - ta = "/".join(f"TA{n - 1}" for n in c) - out.update(status="read", genotype=genotype, ta_units=c) - reading = f"{genotype} ({ta}, {c[0]}/{c[1]} TA units as ExpansionHunter counts them): " - if not all(n in UGT1A1_ALLELES for n in c): - # A length with no named allele has no function to read, and so no - # count of decreased-function copies to set beside PharmCAT's. - return dict(out, pharmcat=".", agreement="not_compared", - reading=reading + "a TA length outside TA5 to TA8 is not a named allele here, so its function " - "is not read and it is not compared with PharmCAT (step 7).") - reading += ("no decreased-function TA allele." if not dec else - f"{dec} decreased-function allele{'s' if dec > 1 else ''} (*28 and *37 are decreased function).") - # PharmCAT's UGT1A1 call (step 27's table, with the reference-block fill - # of bin/pgx_refblock_fill.py): its count of *28 and *37 copies. - pc, agreement = ".", "not_compared" - p = first_existing(d, [f"cpic/{s}_phenotypes.tsv"]) if cpic_state not in ("failed", "unreadable") else None - row = None - if p: - try: - row = next((r for r in read_tsv(p) if r.get("Gene") == "UGT1A1"), None) - except (OSError, csv.Error): - row = None - if row: - pc = (row.get("Diplotype") or "").strip() or "." - alleles = pc.split("/") - if len(alleles) == 2 and "possible" not in pc and (row.get("Status") or "") != "not called": - # A combination allele reads "*80+*28" or "[*80 + *28]". - n = sum(any(x in re.split(r"[+ \[\]]", a) for x in UGT1A1_DECREASED) for a in alleles) - agreement = "agrees" if n == dec else "differs" - old = " (step 27's table is from an earlier run)" if cpic_state == "stale" and agreement != "not_compared" else "" - if agreement == "agrees": - reading += f" PharmCAT (step 7) called {pc}: the same number of *28 and *37 copies{old}." - elif agreement == "differs": - reading += (f" PharmCAT (step 7) called {pc}, a different number of *28 and *37 copies{old}: read the reads " - "at chr2:233760234 before using either.") - elif cpic_state in ("failed", "unreadable"): - reading += f" Not compared with PharmCAT (step 27's table is {OTHER_STEP_STATE[cpic_state]})." - else: - reading += " Not compared with PharmCAT (no unambiguous UGT1A1 call from step 27)." - return dict(out, pharmcat=pc, agreement=agreement, reading=reading) - - -def str_readings(by, d, s, states=None): - """What the counts of the extra repeats mean, one entry per locus, and the - 5-HTTLPR line that no step can test. states: the traits and cpic section - states, once collect() knows them (a stale, failed or unreadable table is - said so, not read as current).""" - states = states or {} - return [cftr_reading(by, d, s, states.get("traits", "ok")), ugt1a1_reading(by, d, s, states.get("cpic", "ok")), - {"locus": "SLC6A4", "repeat": "5-HTTLPR promoter length polymorphism", "status": "not_tested", - "genotype": ".", "reading": HTTLPR_READING, "source": "Bousman et al. 2023, PMID 37032427"}] - - -def sec_expansions(d, s): - eh = first_existing(d, [f"expansion_hunter/{s}_eh.vcf", f"expansion_hunter/{s}.vcf", - "expansion_hunter/*_eh.vcf"]) - st = first_existing(d, [f"expansion_hunter/{s}_eh_stranger.vcf", "expansion_hunter/*_eh_stranger.vcf"]) - # Stranger (step 9b) copies ExpansionHunter's records and adds STR_STATUS. - # Its file is read unless it is older than the ExpansionHunter file it - # would have been made from; then the report says to rerun step 9b. - outdated = bool(st and eh and os.path.getmtime(st) < os.path.getmtime(eh)) - if outdated: - st = None - p = st or eh - if not p: - return None, {} - ranges = str_ranges() - rfc1 = rfc1_motif_row(d, s, st) if st else None - motifs = str_motif_rows(d, s, st) if st else {} - ungraded = str_not_graded(d, s) if st else {} - loci, tested, flagged, no_status, not_graded = [], 0, [], 0, [] - for r in vcf_records(p): - if len(r) < 10: - continue - tested += 1 - info = info_map(r[7]) - # A locus of several repeats (CFTR: CFTR_TG and CFTR_T) writes one - # record per repeat, each with the locus as REPID and its own VARID: - # the record is named by its VARID and looked up by that name in the - # ranges and motif tables (the pathogenic repeat of a multi-repeat - # locus carries the locus name), and in the catalog's NoThresholds by - # its locus (REPID). A one-repeat locus has VARID = REPID. - lid = info.get("REPID") or info.get("VARID") or f"{r[0]}:{r[1]}" - rep = info.get("VARID") or lid - fmt = r[8].split(":") - val = r[9].split(":") - repcn = str_field(fmt, val, "REPCN") - counts = repcn.split("/") if repcn and repcn != "." else [] - n = max(len(counts), 1) - path_min = str_int(info.get("STR_PATHOLOGIC_MIN")) - sizing = [str_allele_sizing(so, ad, ci, cn, path_min) for so, ad, ci, cn in - zip(str_split(str_field(fmt, val, "SO"), n), str_split(str_field(fmt, val, "ADSP"), n), - str_split(str_field(fmt, val, "REPCI"), n), counts or ["."])] - status = [x for x in info.get("STR_STATUS", "").split(",") if x and x != "."] if st else [] - # Stranger writes '.' where its catalog has no value. - item = {"locus": rep, "repeat_count": repcn or ".", "status": ",".join(status), - "normal_max": info.get("STR_NORMAL_MAX", "").strip("."), - "pathologic_min": info.get("STR_PATHOLOGIC_MIN", "").strip("."), - "support": str_field(fmt, val, "SO"), "adsp": str_field(fmt, val, "ADSP"), - "ci": str_field(fmt, val, "REPCI"), "sizing": min(sizing, key=STR_SIZING.index), - "allele_sizing": "/".join(sizing), "zone": "", "zone_label": "", "zone_citation": ""} - rng = ranges.get(rep) - sized = [str_int(c) for c, z in zip(counts, sizing) if z in ("sized", "unknown") and str_int(c) is not None] - if rng and any(int(rng["intermediate_min"]) <= c <= int(rng["intermediate_max"]) for c in sized): - item.update(zone=f"{rng['intermediate_min']}-{rng['intermediate_max']}", zone_label=rng["label"], - zone_citation=rng["citation"]) - if rep == "RFC1" and st and any(x != "normal" for x in status): - item["rfc1_motif"] = rfc1 or {"verdict": "not_checked", - "note": "no motif count beside Stranger's output (step 9b runs it " - "with the sample's BAM): confirm the flag with repeat-primed PCR"} - if rep != "RFC1" and rep in motifs and any(x != "normal" for x in status): - item["motif"] = motifs[rep] - if st and not status and lid in ungraded: - item["not_graded"] = ungraded[lid] - loci.append(item) - if st: - # One value per record, the most severe of normal, pre_mutation and - # full_mutation; a locus outside Stranger's catalog has none. - if not status and "not_graded" in item: - if lid not in not_graded: - not_graded.append(lid) - elif not status: - no_status += 1 - elif any(x != "normal" for x in status): - f = {"locus": item["locus"], "repeat_count": item["repeat_count"], "status": item["status"], - "sizing": item["sizing"]} - if "rfc1_motif" in item: - f["rfc1_motif"] = item["rfc1_motif"] - if "motif" in item: - f["motif"] = item["motif"] - flagged.append(f) - by = {l["locus"]: l for l in loci} - # Every locus, the five key loci first, then by name. - loci.sort(key=lambda l: (EH_LOCI.index(l["locus"]) if l["locus"] in EH_LOCI else len(EH_LOCI), l["locus"])) - out = {"records": tested, "key_loci": [{"locus": k, "repeat_count": by[k]["repeat_count"] if k in by - else "not in output"} for k in EH_LOCI], - "loci": loci, "stranger": bool(st), "stranger_outdated": outdated, - "unsized": [l["locus"] for l in loci if l["sizing"] not in ("sized", "unknown")], - "intermediate": [l["locus"] for l in loci if l["zone"]], - "readings": str_readings(by, d, s)} - if st: - # not_graded: {locus: why} of the loci the catalog gives no thresholds, - # so Stranger gives them no status: not graded, never normal. - out.update(flagged=flagged, no_status=no_status, not_graded={k: ungraded[k] for k in not_graded}) - return p, out - - -# TelomereHunter's estimate rests on its intratelomeric reads. Below this many -# per 30x of depth, about 30 times fewer than a typical 30x short-read genome -# has, the value is too thin to compare (a library or platform problem, or -# reads too short for its repeat threshold). The floor scales with the depth -# TelomereHunter itself counted (total_reads x read length / GENOME_BASES). -TELOMERE_FLOOR_PER_30X = 2000 -GENOME_BASES = 3.1e9 - - -def sec_telomere(d, s): - p = first_existing(d, [f"telomere/{s}/{s}/{s}_summary.tsv", f"telomere/{s}/{s}_summary.tsv", - "telomere/**/*_summary.tsv"]) - if not p: - return None, {} - with open(p, errors="replace") as f: - lines = [l.rstrip("\n").split("\t") for l in f if l.strip()] - if len(lines) < 2: - return p, {"tel_content": "."} - head, row = lines[0], lines[1] - # By header name; older TelomereHunter versions put it in column 11. - i = head.index("tel_content") if "tel_content" in head else 10 - out = {"tel_content": row[i] if i < len(row) else "."} - - def col(*names): - for n in names: - if n in head and head.index(n) < len(row): - return row[head.index(n)] - return None - # TelomereHunter 1.1.0 names it intratel_reads. - intratel = num(col("intratel_reads", "intratelomeric_reads")) - total = num(col("total_reads")) - lengths = [num(x) for x in re.split(r"[,;| ]+", col("read_lengths", "read_length") or "")] - lengths = [x for x in lengths if x] - if intratel is None: - return p, dict(out, read_check="no intratelomeric read count in the summary") - out["intratelomeric_reads"] = int(intratel) - if not total or not lengths: - return p, dict(out, read_check="depth unknown (no total_reads or read_lengths in the summary): not checked") - depth = total * max(lengths) / GENOME_BASES - floor = TELOMERE_FLOOR_PER_30X * depth / 30 - out.update(depth=round(depth, 1), read_floor=int(round(floor)), low_reads=intratel < floor) - return p, out - - -# GRCh38's centromeres and heterochromatin: the acen, gvar and stalk rows of -# UCSC's cytoBand.hg38.txt (the file setup.sh installs as -# reference/cytoBand.hg38.txt, sha256 e514e48b... in scripts/lib/common.sh), -# neighbouring rows merged, as chrom:start-end with a 0-based start and an -# exclusive end. Reads there map badly, so a run of homozygosity or a copy -# number call over them is mostly an artefact. Regenerate with -# gzip -cd cytoBand.txt.gz | awk '$1 ~ /^chr([0-9]+|X|Y)$/ && $5 ~ /^(acen|gvar|stalk)$/' \ -# | sort -k1,1V -k2,2n | awk '{ if ($1 == c && $2 == e) e = $3; else { if (c != "") printf "%s:%d-%d ", c, s, e; -# c = $1; s = $2; e = $3 } } END { print c ":" s "-" e }' -# scripts/11-roh-analysis.sh and modules/local/roh/main.nf carry the same -# line (tests/test_collect_summary.py checks that the three agree). -MASKED_BANDS = ("chr1:121700000-143200000 chr2:91800000-96000000 chr3:87800000-98600000 chr4:48200000-51800000 " - "chr5:46100000-51400000 chr6:58500000-62600000 chr7:58100000-62100000 chr8:43200000-47200000 " - "chr9:42200000-61500000 chr10:38000000-41600000 chr11:51000000-55800000 chr12:33200000-37800000 " - "chr13:0-18900000 chr14:0-18200000 chr15:0-20500000 chr16:35300000-47000000 " - "chr17:22700000-27400000 chr18:15400000-21500000 chr19:19900000-31900000 chr20:25700000-30400000 " - "chr21:0-13000000 chr22:0-17400000 chrX:58100000-63800000 chrY:10300000-10600000 " - "chrY:26600000-57227415") - - -def masked_bands(text=MASKED_BANDS): - """{chrom: [(start, end), ...]} of MASKED_BANDS, 0-based, end exclusive.""" - out = {} - for x in text.split(): - c, _, se = x.partition(":") - a, _, b = se.partition("-") - out.setdefault(c, []).append((int(a), int(b))) - return out - - -def masked_overlap(bands, chrom, start, end): - """Bases of the 1-based, inclusive span start-end inside the masked bands.""" - return sum(max(0, min(end, b) - max(start - 1, a)) for a, b in bands.get(chrom, ())) - - -# GRCh38's chr1 to chr22 (UCSC hg38.chrom.sizes), and the part of them outside -# MASKED_BANDS: the length F_ROH is a share of, since a run's part in a band is -# not counted either. -AUTOSOME_BP = 2875001522 -ROH_MIN_BP = 5e6 # the segments F_ROH and the one-chromosome flag count (docs/11-roh-analysis.md) -# One chromosome carrying most of the long runs: at least UPD_MIN_BP of it in -# segments of 5 Mb or more, and more than UPD_SHARE of the length of all such -# autosomal runs. -# Our thresholds: a run over 20 Mb on one chromosome is the long-standing sign -# of isodisomy, and related parents spread their runs over many chromosomes. -UPD_MIN_BP = 20e6 -UPD_SHARE = 0.5 -# The rows of docs/11-roh-analysis.md's table: F of a child of the parents. -# A value is placed at the row whose F is nearest on a log scale (the -# boundaries are the geometric means of neighbouring rows), and under 1/128 -# (half the second-cousin F) as no sign of related parents. -ROH_BANDS = [(1 / 128, "no sign of related parents"), - ((1 / 64 * 1 / 32) ** 0.5, "second cousins (F 1/64)"), - ((1 / 32 * 1 / 16) ** 0.5, "first cousins once removed (F 1/32)"), - ((1 / 16 * 1 / 8) ** 0.5, "first cousins (F 1/16)"), - ((1 / 8 * 1 / 4) ** 0.5, "half siblings, uncle and niece, double first cousins (F 1/8)"), - (float("inf"), "parent and child, full siblings (F 1/4)")] - -# assets/imprinted_regions.tsv, the same rows (tests/test_collect_summary.py -# checks that the two agree). Held here too because the Nextflow report task -# sees bin/ but not assets/. (chrom, start, end, region, maternal UPD, -# paternal UPD, PMIDs); start and end None for the whole chromosome. -IMPRINTED_REGIONS = [ - ("chr6", None, None, "6q24 (PLAGL1, HYMAI)", "-", "Transient neonatal diabetes mellitus (6q24-related)", - "20301706"), - ("chr7", None, None, "7p12 (GRB10), 7q32 (MEST)", "Silver-Russell syndrome", "-", "27585961"), - ("chr11", 0, 22000000, "11p15 (H19/IGF2, KCNQ1OT1)", "Silver-Russell syndrome", "Beckwith-Wiedemann syndrome", - "29377879;27585961"), - ("chr14", None, None, "14q32.2 (DLK1-MEG3)", "Temple syndrome", "Kagami-Ogata syndrome", "24891339;26377239"), - ("chr15", 20500000, 33400000, "15q11-q13 (SNRPN, UBE3A)", "Prader-Willi syndrome", "Angelman syndrome", - "22237428;27615419"), - ("chr20", None, None, "20q13.32 (GNAS)", "Mulchandani-Bhoj-Conlin syndrome (growth failure)", - "Pseudohypoparathyroidism type 1B", "26248010;29959430"), -] - - -def autosome_unmasked_bp(bands=None): - """AUTOSOME_BP without the autosomal MASKED_BANDS.""" - bands = masked_bands() if bands is None else bands - return AUTOSOME_BP - sum(b - a for c, spans in bands.items() if c[3:] in AUTOSOMES for a, b in spans) - - -def roh_band(f): - """The row of docs/11-roh-analysis.md's table an F_ROH is nearest to.""" - return next(label for top, label in ROH_BANDS if f < top) - - -def imprinted_rows(chrom, segs): - """The IMPRINTED_REGIONS rows a flagged chromosome joins: its own - chromosome's rows, and of a row with a region only when one of the runs - (1-based, inclusive start-end) overlaps it.""" - out = [] - for c, a, b, region, mat, pat, pmid in IMPRINTED_REGIONS: - if c != chrom: - continue - if a is not None and not any(masked_overlap({c: [(a, b)]}, c, s, e) for s, e in segs): - continue - out.append({"region": region, "maternal_upd": mat, "paternal_upd": pat, "pmid": pmid}) - return out - - -def sec_roh(d, s): - """Step 11's bcftools roh output. Segments on chrX and chrY are left out - (a man's single X is one long run), segments on any other contig that is - not chr1 to chr22 (chrM, chrEBV, chrUn_*) are not counted, and so is the - part of a segment that lies in a centromere or heterochromatin band - (MASKED_BANDS), where few markers map. Total, Largest and the list of segments over 5 Mb are of the - autosomal length that is left.""" - p = first_existing(d, [f"vcf/{s}_roh.txt", f"roh/{s}_roh.txt", "roh/*_roh.txt"]) - if not p: - return None, {} - bands = masked_bands() - total = largest = masked = 0.0 - segs, sex_segs, big = 0, 0, [] - per_chrom = {} # chrom -> [long bp, [(start, end), ...]] of the segments over ROH_MIN_BP - with open(p, errors="replace") as f: - for line in f: - c = line.split() - if len(c) < 6 or c[0] != "RG": - continue - try: - start, end, length = int(c[3]), int(c[4]), float(c[5]) - except ValueError: - continue - if not math.isfinite(length): - raise Unreadable(f"{os.path.basename(p)} has a run of length {c[5]}") - if c[2] in ("chrX", "chrY", "X", "Y"): - sex_segs += 1 - continue - if not (c[2].startswith("chr") and c[2][3:] in AUTOSOMES): - continue - inside = min(length, masked_overlap(bands, c[2], start, end)) - masked += inside - length -= inside - if length <= 0: - continue - segs += 1 - total += length - largest = max(largest, length) - if length >= ROH_MIN_BP: - pc = per_chrom.setdefault(c[2], [0.0, []]) - pc[0] += length - pc[1].append((start, end)) - row = {"region": f"{c[2]}:{c[3]}-{c[4]}", "mb": round(length / 1e6, 1)} - if inside: - row["masked_mb"] = round(inside / 1e6, 1) - big.append(row) - long_bp = sum(v[0] for v in per_chrom.values()) - f_roh = long_bp / autosome_unmasked_bp(bands) - upd = None - if per_chrom: - chrom, (bp, spans) = max(per_chrom.items(), key=lambda kv: kv[1][0]) - if bp >= UPD_MIN_BP and bp > UPD_SHARE * long_bp: - upd = {"chrom": chrom, "mb": round(bp / 1e6, 1), "share": round(bp / long_bp, 2), - "imprinted": imprinted_rows(chrom, spans)} - return p, {"segments": segs, "total_mb": round(total / 1e6, 1), "largest_mb": round(largest / 1e6, 1), - "autosomal_over_5mb": big, "masked_mb": round(masked / 1e6, 1), "sex_chromosome_segments": sex_segs, - "long_mb": round(long_bp / 1e6, 1), "f_roh": round(f_roh, 4), - # a chromosome that may come from one parent fills F_ROH by itself, so no band then - "relationship": None if upd else roh_band(f_roh), "upd_flag": upd} - - -# Pages that show more of a called branch, free and with no upload. The URL -# patterns were checked live on 2026-10-25 (docs/37-y-haplogroup.md, -# "Explore your branch further"): FamilyTreeDNA Discover answers its own -# HaplogroupSearch for R-M269 and H5a7, and YFull's tree pages answer 200 for -# a branch it has and 404 for one it does not. -EXPLORE_DOCS = "https://github.com/GeiserX/OwnGenome/blob/main/docs/" - - -def haplogroup_explore(kind, hg): - """[{name, url}] for a called branch: kind 'y' (Yleaf, e.g. R-M269 or - R-M269*(xR-L23)) or 'mt' (haplogrep3, e.g. H5a7). Empty when there is no call.""" - from urllib.parse import quote - hg = (hg or "").strip() - if kind == "y": - hg = hg.split("*", 1)[0] # Yleaf's paragroup mark: the branch is the name before '*' - if not hg or hg in (".", "NA", "insufficient markers"): - return [] - q = quote(hg, safe="") - if kind == "y": - return [{"name": "FamilyTreeDNA Discover (free, no upload)", "url": f"https://discover.familytreedna.com/y-dna/{q}"}, - {"name": "YFull YTree", "url": f"https://www.yfull.com/tree/{q}/"}, - {"name": "More routes, and the files to share", "url": EXPLORE_DOCS + "37-y-haplogroup.md#explore-your-branch-further"}] - return [{"name": "FamilyTreeDNA Discover (free, no upload)", "url": f"https://discover.familytreedna.com/mtdna/{q}"}, - {"name": "YFull MTree", "url": f"https://www.yfull.com/mtree/{q}/"}, - {"name": "More routes, and the files to share", "url": EXPLORE_DOCS + "12-mito-haplogroup.md#explore-your-branch-further"}] - - -def sec_haplogroup(d, s): - p = first_existing(d, [f"mito/{s}_haplogroup.txt"]) - if not p: - return None, {} - rows = read_tsv(p) - r = rows[0] if rows else {} - hg = (r.get("Haplogroup") or (list(r.values())[1] if len(r) > 1 else "") or "").strip('"') - out = {"haplogroup": hg or ".", "explore": haplogroup_explore("mt", hg)} - # haplocheck (step 12 with step 20's Mutect2 calls): a second haplogroup - # in the allele fractions is another person's DNA in the sample. - hc = first_existing(d, [f"mito/{s}_haplocheck.txt"]) - if hc: - rows = [{k.strip('"'): (v or "").strip('"') for k, v in row.items() if k} for row in read_tsv(hc)] - if not rows or "Contamination Status" not in rows[0]: - raise Unreadable(f"{os.path.basename(hc)} has no Contamination Status column") - out["contamination_status"] = rows[0]["Contamination Status"] or "." - out["contamination_level"] = rows[0].get("Contamination Level") or "." - return p, out - - -def sec_y_haplogroup(d, s): - p = first_existing(d, [f"y_haplogroup/{s}_y_haplogroup.txt"]) - if not p: - return None, {} - rows = read_tsv(p) - if not rows or "Hg" not in rows[0]: - raise Unreadable(f"{os.path.basename(p)} has no Hg column or no sample row") - r = rows[0] - hg = (r.get("Hg") or "").strip() - return p, {"haplogroup": hg if hg not in ("", "NA") else "insufficient markers", - "valid_markers": r.get("Valid_markers") or ".", "qc_score": r.get("QC-score") or ".", - "explore": haplogroup_explore("y", hg)} - - -def sec_y_tree(d, s): - """Step 37's placement on the current public tree (y_haplogroup/_y_tree.txt, - og y tree-place): the terminal branch, how deep it sits, the SNP - calls along the path, and the tree release the names come from. The - per-level table beside it gives each branch of that path.""" - p = first_existing(d, [f"y_haplogroup/{s}_y_tree.txt"]) - if not p: - return None, {} - rows = read_tsv(p) - need = ("Hg", "Level", "Tree") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - r = rows[0] - hg = (r.get("Hg") or "").strip() - out = {"haplogroup": hg if hg not in ("", "NA") else "no branch has a derived call", - "level": r.get("Level") or ".", "tree": r.get("Tree") or ".", - "path_derived": r.get("Path_derived") or ".", "path_ancestral": r.get("Path_ancestral") or ".", - "path_no_call": r.get("Path_no_call") or ".", - "ancestral_in_terminal": r.get("Ancestral_in_terminal") or ".", - "outvoted_ancestral": r.get("Outvoted_ancestral") or ".", - "snps_called": r.get("SNPs_called") or ".", "min_depth": r.get("Min_depth") or ".", - "explore": haplogroup_explore("y", hg)} - lvl = first_existing(d, [f"y_haplogroup/{s}_y_tree_path.tsv"]) - if lvl: - # og y tree-place writes the header alone when no branch is derived: - # an empty path, not an unreadable file. - with open_text(lvl) as f: - rd = csv.DictReader(f, delimiter="\t") - prows = list(rd) - header = rd.fieldnames or [] - if "Branch" not in header: - raise Unreadable(f"{os.path.basename(lvl)} has no Branch column") - out["path"] = [{k: (row.get(k) or "") for k in - ("Level", "Branch", "SNPs", "On_tables", "Derived", "Ancestral", "No_call", - "Ancestral_SNPs")} for row in prows] - return p, out - - -def sec_y_str(d, s): - """Step 45's table (y_str/_y_str.tsv, og y str-report): one row per - FamilyTreeDNA marker with its count and confidence, and the counts per - confidence.""" - p = first_existing(d, [f"y_str/{s}_y_str.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = ("order", "panel", "marker", "count", "confidence") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - keep = [{k: r.get(k) or "" for k in need} for r in rows] - counts = {c: sum(1 for r in keep if r["confidence"] == c) for c in ("high", "low", "no call", "not callable")} - if sum(counts.values()) != len(keep): - raise Unreadable(f"{os.path.basename(p)} has a confidence other than high, low, no call or not callable") - return p, {"markers": keep, **{k.replace(" ", "_"): v for k, v in counts.items()}} - - -def sec_consumer_export(d, s): - """Step 40's summary table (key, value): how many chip sites got a genotype.""" - p = first_existing(d, [f"consumer_export/{s}_consumer_export.tsv"]) - if not p: - return None, {} - data = {r["key"]: r.get("value", "") for r in read_tsv(p) if r.get("key")} - for k in ("sites", "called", "no_call"): - if not (data.get(k) or "").isdigit(): - raise Unreadable(f"{os.path.basename(p)} has no whole-number '{k}' row") - reasons = {} - for k, v in data.items(): - if k.startswith("no_call: "): - if not (v or "").isdigit(): - raise Unreadable(f"{os.path.basename(p)} has a '{k}' row without a whole-number count") - reasons[k[len("no_call: "):]] = int(v) - return p, {"sites": int(data["sites"]), "called": int(data["called"]), "no_call": int(data["no_call"]), - "from_reference_blocks": int(data.get("from_reference_blocks") or 0), - "template_sites": data.get("template_sites") or ".", - "file_23andme": data.get("file_23andme") or ".", - "file_ancestrydna": data.get("file_ancestrydna") or ".", - "no_call_reasons": reasons} - - -def sec_bloodtype(d, s): - """Step 41's table: one row per blood group system, predicted from the genome.""" - p = first_existing(d, [f"bloodtype/{s}_bloodtype.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - if not rows or not {"system", "phenotype", "confidence"} <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} has no system, phenotype and confidence columns, or no row") - systems = [{"system": r["system"], "phenotype": r["phenotype"] or ".", "genotype": r.get("genotype") or ".", - "confidence": r["confidence"] or "."} for r in rows] - by = {x["system"]: x["phenotype"] for x in systems} - return p, {"systems": systems, "abo": by.get("ABO", "."), "rhd": by.get("RhD", ".")} - - -def sec_archaic(d, s): - """Step 43's summary (archaic/_archaic.tsv, bin/archaic_report.py): the - Neanderthal and Denisovan shares and segment counts, the analysed span and - the ten longest segments.""" - p = first_existing(d, [f"archaic/{s}_archaic.tsv"]) - if not p: - return None, {} - data = {r["key"]: r.get("value") or "" for r in read_tsv(p) if r.get("key")} # a short row: None - out = {"population": data.get("population", "."), - "longest": [data[k] for k in sorted((k for k in data if re.fullmatch(r"longest_[0-9]+", k)), - key=lambda k: int(k.split("_")[1]))]} - for k in ("neanderthal_segments", "neanderthal_bp", "denisovan_segments", "denisovan_bp", "analysed_bp"): - if not re.fullmatch(r"[0-9]+", data.get(k, "")): - raise Unreadable(f"{os.path.basename(p)} has no whole number for {k}") - out[k] = int(data[k]) - for k in ("neanderthal_percent", "denisovan_percent"): - if not re.fullmatch(r"[0-9]+\.[0-9]+", data.get(k, "")): - raise Unreadable(f"{os.path.basename(p)} has no percentage for {k}") - out[k] = data[k] - return p, out - - -# The two LPA risk variants of assets/traits.tsv (rsid -> its alt, the allele -# that goes with a high Lp(a)); tests/test_lpa_kiv2.py checks they agree. -LPA_RISK_ALLELES = {"rs10455872": "G", "rs3798220": "C"} - - -def sec_lpa(d, s): - """Step 46's KIV-2 count (lpa/_lpa_kiv2.tsv, og cn call --locus LPA_KIV2), with - step 42's two LPA risk variants when the traits table is there, and - whether either points to a genetically high Lp(a).""" - p = first_existing(d, [f"lpa/{s}_lpa_kiv2.tsv"]) - if not p: - return None, {} - kv = {r.get("metric"): r.get("value") for r in read_tsv(p)} - if kv.get("status") not in ("estimated", "too little depth", "not assessed"): - raise Unreadable(f"{os.path.basename(p)} has no status row (estimated, too little depth or not assessed)") - data = {k: kv.get(k) or "." for k in ("status", "kiv2_copies", "ci95_low", "ci95_high", "class", - "kiv2_depth", "flank_depth", "mapq0_share", "message", "advice")} - # KIV-2 4925G>A beside the count (step 46, lpa/_lpa_kiv2_4925.tsv) - data["null_allele"] = read_marker_table(os.path.join(os.path.dirname(p), f"{s}_lpa_kiv2_4925.tsv"), "46", - "LPA_KIV2_4925") - variants = [] - t = first_existing(d, [f"traits/{s}_traits.tsv"]) - for r in (read_tsv(t) if t else []): - alt = LPA_RISK_ALLELES.get(r.get("rsid")) - if not alt: - continue - called = r.get("status") == "called" - n = re.split(r"[/|]", r.get("genotype") or "").count(alt) if called else None - variants.append({"rsid": r["rsid"], "genotype": r.get("genotype") or ".", "status": r.get("status") or ".", - "risk_alleles": n}) - risk = sum(v["risk_alleles"] for v in variants if v["risk_alleles"] is not None) - reasons = ([f"{risk} risk allele{'s' if risk != 1 else ''} at rs10455872 and rs3798220"] if risk else []) \ - + (["a low KIV-2 repeat count"] if data["class"] == "low" else []) - read = data["status"] == "estimated" or any(v["risk_alleles"] is not None for v in variants) - data.update(variants=variants, risk_alleles=risk, flag="high" if reasons else ".", - flag_text=("Genetic signs of a high Lp(a): " + " and ".join(reasons) + ". Measure Lp(a) in blood." - if reasons else - "No genetic sign of a high Lp(a) from these markers. They explain only part of it: " - "the KIV-2 count explained 21% to 27% of the variation in a Danish population study." - if read else - "Not determined: neither the KIV-2 count nor the two risk variants could be read. " - "Measure Lp(a) in blood.")) - return p, data - - -VNTYPER_STATUSES = ("positive", "possible", "negative", "not assessed") - - -def sec_vntyper(d, s): - """Step 52's table (vntyper/_vntyper.tsv, og vntyper report): - VNtyper's MUC1 VNTR reading with its depth and sensitivity line.""" - p = first_existing(d, [f"vntyper/{s}_vntyper.tsv"]) - if not p: - return None, {} - kv = {r.get("metric"): r.get("value") for r in read_tsv(p)} - if kv.get("status") not in VNTYPER_STATUSES: - raise Unreadable(f"{os.path.basename(p)} has no status row ({', '.join(VNTYPER_STATUSES)})") - return p, {k: kv.get(k) or "." for k in ("status", "confidence", "variant", "flag", "alt_depth", - "vntr_mean_depth", "coverage_qc", "sensitivity", "message", "advice")} - - -LOCITYPER_COLUMNS = ("locus", "region", "status", "genotype", "haplotype_1", "haplotype_2", "quality", - "total_reads", "unexplained_reads", "weighted_distance", "warnings", "filter", "reason") - - -def sec_locityper(d, s): - """Step 53's table (locityper/_locityper.tsv, og locityper report): - one row per locus of assets/locityper_loci.bed, the two pangenome - haplotypes Locityper picked with its quality and the authors' filter. - collect() adds each locus's primary call (locityper_primary).""" - p = first_existing(d, [f"locityper/{s}_locityper.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - if not rows or not set(LOCITYPER_COLUMNS) <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} lacks a row or one of the columns {', '.join(LOCITYPER_COLUMNS)}") - loci = [{k: (r.get(k) or ".") for k in LOCITYPER_COLUMNS} for r in rows] - return p, {"loci": loci, "called": sum(x["status"] == "called" for x in loci), - "passed": sum(x["filter"] == "pass" for x in loci)} - - -def locityper_primary(sections): - """The primary caller's call at each Locityper locus, as text, so the - report shows the two side by side: T1K (step 08) for HLA-DRB1 and - HLA-DQB1, T1K's KIR pass (KIR=true) for KIR, pypgx (step 32) and Cyrius - (step 21) for CYP2D6. FCGR has no primary caller here. Locityper names - pangenome haplotypes, not alleles, so the two are shown, not compared.""" - def old(key): # a table older than the last run-all.sh run says so - return " (stale: from an earlier run)" if sections[key]["state"] == "stale" else "" - - hla = sections["hla"]["data"] if sections["hla"]["state"] in ("ok", "stale") else {} - by_gene = {x["gene"]: x for x in hla.get("loci", [])} - out = {} - for g in ("HLA-DRB1", "HLA-DQB1"): - x = by_gene.get(g) or by_gene.get(g.replace("HLA-", "")) - out[g] = ("T1K (step 08): " + ("/".join(x["alleles"]) or "no allele") - + (" (low confidence)" if x.get("low_confidence") else "") + old("hla")) if x else "T1K (step 08) did not run" - kir = hla.get("kir") - out["KIR"] = (f"T1K KIR pass (step 08): {len(kir['typed'])} genes typed{old('hla')}" if kir and kir.get("typed") - else "T1K's KIR pass (step 08, KIR=true) gave no call") - cyp = [] - if sections["pypgx"]["state"] in ("ok", "stale"): - cyp.append(f"pypgx (step 32): {sections['pypgx']['data'].get('cyp2d6', 'not called')}{old('pypgx')}") - if sections["cyrius"]["state"] in ("ok", "stale"): - cy = sections["cyrius"]["data"] - usable = "" if cy.get("filter") == "PASS" else f" (not usable: Filter {cy.get('filter') or 'missing'})" - cyp.append(f"Cyrius (step 21): {cy.get('genotype', 'none')}{usable}{old('cyrius')}") - out["CYP2D6"] = "; ".join(cyp) or "pypgx (step 32) and Cyrius (step 21) did not run" - out["FCGR"] = "no other step calls FCGR" - return out - - -def sec_viral(d, s): - """Step 47's table (viral/_viral.tsv, og viral report): one row per - viral genome; the HHV-6 line and the genomes with reads (phiX included).""" - p = first_existing(d, [f"viral/{s}_viral.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = ("virus", "accession", "reads", "breadth", "relative_depth", "copies_per_cell", "call") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - keep = [{k: r[k] or "." for k in need + ("human_depth", "note")} for r in rows] - ici = [r["virus"] for r in keep if r["call"] == "inherited integrated HHV-6 likely"] - # No read on chr1 to chr22 leaves no human depth, so integration was never assessed. - hhv6 = (f"inherited integrated {ici[0]} likely" if ici - else "not assessed (no human depth)" if keep[0]["human_depth"] == "." - else "no integrated HHV-6") - return p, {"hhv6": hhv6, - "human_depth": keep[0]["human_depth"], - "found": [r for r in keep if r["reads"].isdigit() and int(r["reads"]) > 0], "genomes": len(keep)} - - -def sec_chip(d, s): - """Step 49's tables (chip/_chip.tsv and chip/_chip_depth.tsv, bin/chip_report.py): - the CHIP calls, how many other protein changes were listed and not counted, - and the depth and detection limit per gene with their median over the - covered genes. The limit is the one at the median depth, as step 49 prints it.""" - p = first_existing(d, [f"chip/{s}_chip.tsv"]) - dp = first_existing(d, [f"chip/{s}_chip_depth.tsv"]) - if not p or not dp: - return None, {} - calls, depth = read_tsv(p), read_tsv(dp) - need = ("gene", "protein_change", "vaf", "alt_reads", "depth", "status", "note") - with open(p, encoding="utf-8", errors="replace") as f: - head = f.readline().rstrip("\n").split("\t") - if any(k not in head for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - if not depth or not {"gene", "mean_depth", "detection_limit_vaf", "status"} <= set(depth[0]): - raise Unreadable(f"{os.path.basename(dp)} has no gene, mean_depth, detection_limit_vaf and status columns, or no row") - chip = [{k: r.get(k) or "." for k in need} for r in calls if r.get("status") == "chip"] - covered = sorted(float(r["mean_depth"]) for r in depth if r["status"] == "covered") - med = None - if covered: - n = len(covered) - med = covered[n // 2] if n % 2 else (covered[n // 2 - 1] + covered[n // 2]) / 2 - return p, {"calls": chip, "other": len(calls) - len(chip), - "genes": [{k: r.get(k) or "." for k in ("gene", "mean_depth", "detection_limit_vaf", "status")} - for r in depth], - "median_depth": med, "median_limit": chip_report.detection_limit(med) if med is not None else None, - "not_covered": [r["gene"] for r in depth if r["status"] != "covered"]} - - -def sec_mei(d, s): - """Step 50's tables (mei/_mei.tsv and _mei_sites.tsv, og mei report): - the insertions MEGAnE calls, counted per element, and every row of the - known-site catalog with seen, not seen or not callable.""" - p = first_existing(d, [f"mei/{s}_mei_sites.tsv"]) - if not p: - return None, {} - sites = read_tsv(p) - need = ("gene", "element", "condition", "inheritance", "status", "mean_depth", "call") - if not sites or any(k not in sites[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - cp = os.path.join(os.path.dirname(p), f"{s}_mei.tsv") - if not os.path.isfile(cp): - raise Unreadable(f"{os.path.basename(cp)} is missing beside {os.path.basename(p)}") - with open_text(cp) as f: # the header alone: a run with no insertion writes no row - r = csv.DictReader(f, delimiter="\t") - head, calls = r.fieldnames or [], list(r) - if any(k not in head for k in ("chrom", "pos", "element", "copies")): - raise Unreadable(f"{os.path.basename(cp)} lacks a column of chrom, pos, element, copies") - counts = {} - for c in calls: - counts[c["element"]] = counts.get(c["element"], 0) + 1 - return p, {"calls": len(calls), "by_element": dict(sorted(counts.items())), - "sites": [{k: r[k] or "." for k in need} for r in sites]} - - -MOCHA_SUMMARY_KEYS = ("status", "message", "depth", "het_sites", "het_per_10mb", "chromosomes", "floor_read", - "floor_min", "floor", "calls_mocha", "calls_reported", "phasing") -MOCHA_CALL_KEYS = ("chrom", "start", "end", "length_mb", "arms", "type", "cell_fraction", "n_hets", "lod_baf_phase", - "reported", "reason") - - -def sec_mocha(d, s): - """Step 51's tables (mosaic/_mocha_summary.tsv and _mocha.tsv, - og mocha report): the status, the floor, and the calls reported.""" - p = first_existing(d, [f"mosaic/{s}_mocha_summary.tsv"]) - if not p: - return None, {} - kv = {r.get("metric"): r.get("value") for r in read_tsv(p)} - if kv.get("status") not in ("assessed", "not assessed"): - raise Unreadable(f"{os.path.basename(p)} has no status row (assessed or not assessed)") - data = {k: kv.get(k) or "." for k in MOCHA_SUMMARY_KEYS} - c = first_existing(d, [f"mosaic/{s}_mocha.tsv"]) - if not c: - raise Unreadable(f"{os.path.basename(p)} has no {s}_mocha.tsv beside it") - with open_text(c) as f: - head = f.readline().rstrip("\r\n").split("\t") - if any(k not in head for k in MOCHA_CALL_KEYS): - raise Unreadable(f"{os.path.basename(c)} lacks a column of {', '.join(MOCHA_CALL_KEYS)}") - rows = read_tsv(c) - data["calls"] = [{k: r[k] or "." for k in MOCHA_CALL_KEYS} for r in rows if r.get("reported") == "yes"] - data["not_reported"] = sum(1 for r in rows if r.get("reported") != "yes") - if data["calls_reported"] != "." and data["calls_reported"] != str(len(data["calls"])): - raise Unreadable(f"{os.path.basename(c)} has {len(data['calls'])} reported calls, " - f"{os.path.basename(p)} says {data['calls_reported']}") - return p, data - - -def sec_traits(d, s): - """Step 42's traits table (traits/_traits.tsv, bin/traits_report.py): - every row with its genotype, status and reading, the counts per status, - and the APOE line on its own (the reports fold it away).""" - p = first_existing(d, [f"traits/{s}_traits.tsv"]) - if not p: - return None, {} - with open(p, encoding="utf-8", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - need = ("rsid", "gene", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - # A row cut short (DictReader fills the missing fields with None) or with - # extra fields (kept under the key None) is a damaged table, never a - # called trait without its reading. - bad = next((i for i, r in enumerate(rows, 2) if None in r or any(r.get(k) is None for k in need)), None) - if bad: - raise Unreadable(f"{os.path.basename(p)} line {bad} does not have one field per column") - keep = [{k: r[k] for k in need} for r in rows] - apoe = [r for r in keep if r["gene"] == "APOE"] - counts = {st: sum(1 for r in keep if r["status"] == st and r["gene"] != "APOE") - for st in ("called", "not_covered", "other_allele")} - return p, {"rows": [r for r in keep if r["gene"] != "APOE"], "apoe": apoe, **counts, - "rules": genotype_rules(d, s)} - - -def genotype_rules(d, s): - """Step 42's genotype rules (traits/_genotype_rules.tsv): one line per - rule that reads several sites together (TAS2R38's haplotypes), or None - when the table is not there (a run from before the rules). The rules - whose line the traits table already holds (APOE) are left out, so the - reports keep folding APOE away.""" - p = first_existing(d, [f"traits/{s}_genotype_rules.tsv"]) - if not p: - return None - with open(p, encoding="utf-8", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - need = traits_report.RULE_OUT_COLUMNS - if rows and any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - bad = next((i for i, r in enumerate(rows, 2) if None in r or any(r.get(k) is None for k in need)), None) - if bad: - raise Unreadable(f"{os.path.basename(p)} line {bad} does not have one field per column") - out = [{k: r[k] for k in need} for r in rows if r["rule"] not in traits_report.TRAITS_TABLE_RULES] - for r in out: - if r["rule"] == genotype_rules_indels.G2["rule"]: - g2 = apol1_g2(d, s) - r["g2"] = dict(g2, **genotype_rules_indels.combine(r, g2)) - return out - - -def vcf_window(path, chrom, start, end): - """The split data lines of a VCF on chrom that can cover start..end: read - from the tabix linear index's offset for start when the .tbi is there, - else the whole file (stopping past the window).""" - out = [] - if os.path.exists(path + ".tbi"): - ioff = tabix_linear_index(path).get(chrom) - if not ioff: - return out - voff = ioff[min((start - 1) >> 14, len(ioff) - 1)] - voff = 0 if voff == 0xFFFFFFFFFFFFFFFF else voff - with open(path, "rb") as raw: - raw.seek(voff >> 16) - g = gzip.GzipFile(fileobj=raw) - g.read(voff & 0xFFFF) - lines = (x.decode(errors="replace").rstrip("\n").split("\t") for x in g if not x.startswith(b"#")) - seen = False - for r in lines: - if len(r) < 2 or not r[1].isdigit(): - continue - if r[0] != chrom: - if seen: - break - continue - seen = True - if int(r[1]) > end: - break - out.append(r) - return out - seen = False - for r in vcf_records(path): - if len(r) < 2 or not r[1].isdigit(): - continue - if r[0] != chrom: - if seen: - break - continue - seen = True - if int(r[1]) > end: - break - out.append(r) - return out - - -def apol1_g2(d, s): - """APOL1 G2 (rs71785313, a 6-base deletion step 42's rule table cannot - read) from the sample's gVCF when it is there (vcf/.g.vcf.gz, whose - reference blocks tell a reference base from no reads), else its VCF - (vcf/.vcf.gz): bin/genotype_rules_indels.py's read_g2, plus source.""" - m = genotype_rules_indels - for name, kind in ((f"vcf/{s}.g.vcf.gz", "gvcf"), (f"vcf/{s}.vcf.gz", "vcf")): - p = os.path.join(d, name) - if not os.path.isfile(p): - continue - try: - recs = vcf_window(p, m.G2["chrom"], m.CONTEXT_START, m.SPAN[1]) - except (OSError, EOFError, ValueError, struct.error, gzip.BadGzipFile, zlib.error, Unreadable) as e: - return {"status": "not_read", "copies": None, "gq": ".", "source": name, - "text": f"G2 (rs71785313) not read: {name} could not be read ({e})."} - return dict(m.read_g2(recs, kind), source=name) - return {"status": "not_read", "copies": None, "gq": ".", "source": None, - "text": "G2 (rs71785313) not read: no VCF of the sample in this folder."} - - -def sec_carrier(d, s): - """The sample's own carrier list (carrier_list) from step 06's ClinVar - hits; its own section, so the ClinVar section reads the same whether or - not the assets folder is there.""" - p = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - if not p: - return None, {} - return p, carrier_list_or_note(p, d, s) - - -def sec_exomiser(d, s): - """Step 48's ranking (exomiser/_exomiser_top.tsv, og exomiser report): - the HPO terms searched, and each rank with its gene, mode of inheritance, - scores and contributing variants. A table without rows means no gene - passed Exomiser's filters.""" - p = first_existing(d, [f"exomiser/{s}_exomiser_top.tsv"]) - if not p: - return None, {} - with open(p, encoding="utf-8", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - f.seek(0) - header = f.readline().rstrip("\n").split("\t") - need = ("rank", "gene", "moi", "exomiser_score", "phenotype_score", "variant", "genotype", "consequence", - "hgvs", "acmg", "disease") - if any(k not in header for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - if any(None in r or any(r.get(k) is None for k in need) for r in rows): - raise Unreadable(f"{os.path.basename(p)} has a line that does not have one field per column") - terms = [] - tp = os.path.join(os.path.dirname(p), f"{s}_exomiser_terms.tsv") - if os.path.isfile(tp): - with open(tp, encoding="utf-8", errors="replace") as f: - terms = [{"id": r.get("hpo_id", ""), "label": r.get("label", "")} - for r in csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)] - genes = [] - for r in rows: - if not genes or genes[-1]["rank"] != r["rank"]: - genes.append({k: r[k] for k in ("rank", "gene", "moi", "exomiser_score", "phenotype_score")}) - genes[-1]["variants"] = [] - genes[-1]["variants"].append({k: r[k] for k in ("variant", "genotype", "consequence", "hgvs", "acmg", "disease")}) - return p, {"terms": terms, "genes": genes} - - -def sec_couple(d, s): - """Step 39's couple table (couple/_couple_carrier.tsv, bin/couple_report.py): - the partner, the shared and X-linked genes with the chance for a child, and - how many genes only one partner carries a change in. A table without rows - (no finding in either partner) names no partner.""" - p = first_existing(d, [f"couple/{s}_couple_carrier.tsv"]) - if not p: - return None, {} - with open(p, encoding="utf-8", errors="replace") as f: - reader = csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE) - rows = list(reader) - cols = reader.fieldnames or [] - need = ("gene", "condition", "result", "offspring_risk", "partner_a", "findings_a", "partner_b", - "findings_b", "explanation") - if any(k not in cols for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - bad = next((i for i, r in enumerate(rows, 2) if None in r or any(r.get(k) is None for k in need) - or s not in (r["partner_a"], r["partner_b"]) or r["result"] not in ("shared", "x_linked", "one_partner")), - None) - if bad: - raise Unreadable(f"{os.path.basename(p)} line {bad} is not a row of {s}'s couple table") - - def side(r): - own = "a" if r["partner_a"] == s else "b" - other = "b" if own == "a" else "a" - return {"gene": r["gene"], "condition": r["condition"], "risk": r["offspring_risk"], - "own": r["findings_" + own], "partner_findings": r["findings_" + other], - "explanation": r["explanation"]} - partner = next((r["partner_b"] if r["partner_a"] == s else r["partner_a"] for r in rows), None) - return p, {"partner": partner, - "shared": [side(r) for r in rows if r["result"] == "shared"], - "x_linked": [side(r) for r in rows if r["result"] == "x_linked"], - "one_partner": sum(1 for r in rows if r["result"] == "one_partner")} - - -# The D-loop poly-C tracts (rCRS m.303-315 and m.16184-16193). Length -# variants there, an extra or missing C, are found in most people, often as a -# mix of lengths that reads as heteroplasmy; they are counted apart and left -# out of the heteroplasmic count. -POLYC_TRACTS = ((303, 315), (16184, 16193)) - - -def mito_span(pos, ref, alt): - """The rCRS positions a chrM allele changes: an SNV or MNV its own bases, - an indel the bases after its anchor base (Mutect2 writes the anchor).""" - if len(ref) == len(alt): - return pos, pos + len(ref) - 1 - return pos + 1, pos + max(len(ref) - 1, 1) - - -def in_polyc(pos, ref, alt): - lo, hi = mito_span(pos, ref, alt) - return any(lo <= b and hi >= a for a, b in POLYC_TRACTS) - - -def min_allele(pos, ref, alt): - """(pos, ref, alt) without the bases ref and alt share at the end and then - at the start, one base always kept: Mutect2's A>AC of a two-ALT record - AC>A,ACC becomes the same key as ClinVar's and gnomAD's.""" - while len(ref) > 1 and len(alt) > 1 and ref[-1] == alt[-1]: - ref, alt = ref[:-1], alt[:-1] - while len(ref) > 1 and len(alt) > 1 and ref[0] == alt[0]: - ref, alt, pos = ref[1:], alt[1:], pos + 1 - return pos, ref, alt - - -MITO_ANNOT_COLS = ["chrom", "pos", "ref", "alt", "filter", "af", "clinvar_id", "clinvar_significance", - "clinvar_review", "clinvar_condition", "gnomad_af_hom", "gnomad_af_het", "gnomad_max_hl"] -# ClinVar classes: P/LP are counted; a record whose every class is benign or -# carries no classification is not listed unless the call is heteroplasmic. -CLINVAR_PLP = ("Pathogenic", "Likely_pathogenic", "Pathogenic/Likely_pathogenic") -CLINVAR_BENIGN = ("Benign", "Likely_benign", "Benign/Likely_benign", "not_provided") - - -def clinvar_terms(sig): - return [t for t in re.split(r"[|,]", sig or "") if t and t != "."] - - -# --- step 20: large mtDNA deletions (eKLIPse) and the mtDNA copy number ---------- - -# _chrM_deletions.tsv (og mito deletions): eKLIPse's deletions, one row per -# deletion. bkp5 and bkp3 are its 5' and 3' breakpoints as it writes them: the -# last rCRS base kept before the deletion and the first one kept after it, so -# the deleted bases are bkp5+1 to bkp3-1 (across the origin, m.16569 to m.1, -# when bkp3 is the smaller). level is the share of the mtDNA -# molecules with the deletion (in %), the mean of the forward and reverse -# strand estimates. -# A deletion of assets/mito_confirmed.tsv is named when both breakpoints lie -# within this many bases of its own: a deletion between two direct repeats -# can be written anywhere along them (the common one's repeats are 13 bp). -MITO_DEL_TOLERANCE = 50 -# The report lists every named deletion and any other at this level or more; -# the rest are counted. -MITO_DEL_LIST_PCT = 1.0 -# The mtDNA copy number: chrM depth over autosomal depth, from the read counts -# of the BAM index (samtools idxstats), times 2 for the two nuclear copies. -# Under this many reads per kb of autosome (about 3x at 150 bp) the BAM is a -# slice, a panel or a shallow genome, and the ratio would mean nothing. An -# exome can pass the floor and is not told apart from a genome here: the -# number is meant for whole-genome BAMs (docs/20-mtoolbox.md). -MITO_CN_COLS = ["status", "reason", "chrm_reads", "chrm_length", "autosomal_reads", "autosomal_length", - "depth_ratio", "mtdna_copies"] - - -def mito_confirmed(): - """(SNVs {(pos, ref, alt): row}, deletions [row]) of assets/mito_confirmed.tsv.""" - snv, dels = {}, [] - for r in asset_rows("mito_confirmed.tsv"): - if r["kind"] == "snv": - snv[(int(r["pos"]), r["ref"], r["alt"])] = r - elif r["kind"] == "deletion": - dels.append(r) - return snv, dels - - -def known_deletion(row, dels): - """The deletion of assets/mito_confirmed.tsv a called deletion is, or None.""" - for k in dels: - if abs(row["bkp5"] + 1 - int(k["pos"])) <= MITO_DEL_TOLERANCE \ - and abs(row["bkp3"] - 1 - int(k["end"])) <= MITO_DEL_TOLERANCE: - return k - return None - - -def _confirmed_fields(k): - return {"hgvs": k["hgvs"], "gene": k["gene"], "disease": k["disease"], "clinvar_id": k["clinvar_id"], - "pmids": k["pmids"], "note": k["heteroplasmy_note"]} - - -def mito_extended(d, s, vcf): - """Step 20's confirmed variants (assets/mito_confirmed.tsv) among the chrM - calls, its large deletions and the mtDNA copy number.""" - out = {} - try: - snv, dels = mito_confirmed() - except Unreadable as e: - snv, dels = None, [] - out["confirmed_note"] = str(e) - confirmed = [] - for r in vcf_records(vcf) if snv is not None else (): - if len(r) < 8: - continue - fmt = r[8].split(":") if len(r) > 9 else [] - val = r[9].split(":") if len(r) > 9 else [] - afs = val[fmt.index("AF")].split(",") if "AF" in fmt and fmt.index("AF") < len(val) else [] - for i, alt in enumerate(r[4].split(",")): - k = snv.get(min_allele(int(r[1]), r[3], alt)) - if k: - confirmed.append(_confirmed_fields(k) | {"pos": int(k["pos"]), "filter": r[6], - "af": afs[i] if i < len(afs) else "."}) - if snv is not None: - out["confirmed"] = confirmed - p = first_existing(d, [f"mito/{s}_chrM_deletions.tsv"]) - if not p: - out["deletions_note"] = ("no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked " - "for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow " - "MITO_DELETIONS task failed (see the Nextflow log)") - else: - with open(p, errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - listed = [] - for r in rows: - row = {"bkp5": int(r["bkp5"]), "bkp3": int(r["bkp3"]), - "length": None if r["length"] == "." else int(r["length"]), "level_pct": float(r["level_pct"])} - k = known_deletion(row, dels) - if k or row["level_pct"] >= MITO_DEL_LIST_PCT: - listed.append(row | ({"known": _confirmed_fields(k)} if k else {})) - out["deletions"] = {"source": os.path.relpath(p, d), "total": len(rows), "listed": listed, - "list_floor_pct": MITO_DEL_LIST_PCT} - p = first_existing(d, [f"mito/{s}_mtdna_copies.tsv"]) - if p: - m = dict(read_tsv_pairs(p)) - out["copies"] = {k: (None if m.get(k) in (None, ".") else m[k]) for k in MITO_CN_COLS} - else: - out["copies_note"] = ("no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted " - "mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)") - return out - - -def read_tsv_pairs(path): - """[(metric, value)] of a two-column table with a header line.""" - with open(path, errors="replace") as f: - lines = [ln.rstrip("\n").split("\t") for ln in f if ln.strip()] - return [(c[0], c[1]) for c in lines[1:] if len(c) >= 2] - - -def mito_annotation(d, s, vcf): - """Step 20's annotation table, as the report lists it, or a note on why - there is none.""" - p = first_existing(d, [f"mito/{s}_chrM_annotated.tsv"]) - if not p: - return {"note": "no annotation table (mito/_chrM_annotated.tsv): step 20 ran before it annotated, " - "or the Nextflow MITO_VARIANTS_ANNOTATE task failed (see the Nextflow log)"} - if os.path.getmtime(p) < os.path.getmtime(vcf): - return {"note": f"{os.path.relpath(p, d)} is older than the chrM calls: rerun step 20"} - sources, lines = {}, [] - with open(p, errors="replace") as f: - for line in f: - if line.startswith("# "): - c = line[2:].rstrip("\n").split("\t") - if len(c) >= 3: - sources[c[0]] = (f"not annotated: {c[2]}" if c[1] == "not annotated" - else c[1] + (f" ({c[2]})" if c[2] != "-" else "")) - elif line.strip(): - lines.append(line) - rows = [r for r in csv.DictReader(lines, delimiter="\t") if r.get("filter") == "PASS"] - return {"source": os.path.relpath(p, d), "clinvar": sources.get("clinvar"), "gnomad": sources.get("gnomad"), - "rows": rows} - - -def sec_mito(d, s): - p = first_existing(d, [f"mito/{s}_chrM_filtered.vcf.gz"]) - if not p: - return None, {} - passed = het = 0 - polyc, sites = [], [] - for r in vcf_records(p): - if len(r) < 10 or r[6] != "PASS": - continue - passed += 1 - fmt = r[8].split(":") - val = r[9].split(":") - if "AF" not in fmt or fmt.index("AF") >= len(val): - continue - try: - afs = [float(x) for x in val[fmt.index("AF")].split(",") if x not in ("", ".")] - af = max(afs) - except ValueError: - continue - # Heteroplasmic: 5% to 95%. Below 5% NUMT reads and noise dominate. - if not 0.05 <= af < 0.95: - continue - alts = r[4].split(",") - if any(in_polyc(int(r[1]), r[3], a) for a in alts): - polyc.append({"variant": f"{r[0]}:{r[1]} {r[3]}>{r[4]}", "af": round(af, 3)}) - else: - het += 1 - # Each counted site, for the report's chart (bin/report_charts.py). - sites.append({"pos": int(r[1]), "ref": r[3], "alt": r[4], "af": round(af, 3)}) - ann = mito_annotation(d, s, p) - listed, plp = [], 0 - for r in ann.pop("rows", []): - terms = clinvar_terms(r.get("clinvar_significance")) - is_plp = any(t in CLINVAR_PLP for t in terms) - plp += is_plp - try: - het_row = 0.05 <= float(r.get("af")) < 0.95 - except (TypeError, ValueError): - het_row = False - if is_plp or het_row or (terms and not all(t in CLINVAR_BENIGN for t in terms)): - listed.append({k: r.get(k) for k in MITO_ANNOT_COLS if k not in ("chrom", "filter")} - | {"polyc": in_polyc(int(r["pos"]), r["ref"], r["alt"])}) - if "source" in ann: - ann.update(variants=listed, clinvar_plp=plp) - return p, {**mito_extended(d, s, p), "pass": passed, "heteroplasmic": het, "heteroplasmic_sites": sites, "heteroplasmy_floor": 0.05, - "polyc_heteroplasmic": polyc, - "polyc_tracts": [f"m.{a}-{b}" for a, b in POLYC_TRACTS], "annotation": ann} - - -def cpsr_cdna_checks(d, s): - """Step 17's cDNA read check of CPSR's P/LP rows (bin/clip_check.py - --cpsr), keyed like step 06's; None when step 17 did not write it.""" - p = first_existing(d, [f"cpsr/{s}_cpsr_cdna_check.tsv"]) - return clinvar_cdna_checks(d, s, p) if p else None - - -def sec_cpsr(d, s): - html = first_existing(d, [f"cpsr/{s}.cpsr.grch38.html", "cpsr/*.cpsr.*.html"]) - tsv = first_existing(d, [f"cpsr/{s}.cpsr.grch38.classification.tsv.gz", "cpsr/*.cpsr.*.classification.tsv.gz"]) - p = tsv or html - if not p: - return None, {} - out = {"html_report": bool(html)} - if tsv: - try: - with gzip.open(tsv, "rt", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - cols = rows[0].keys() if rows else [] - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(tsv)}: {e}") from e - col = next((c for c in ("CLASSIFICATION", "FINAL_CLASSIFICATION", "CPSR_CLASSIFICATION") if c in cols), None) - if col: - counts = {} - for r in rows: - counts[r.get(col) or "."] = counts.get(r.get(col) or ".", 0) + 1 - out["classification_column"] = col - out["classification"] = dict(sorted(counts.items(), key=lambda kv: -kv[1])) - out.update(cpsr_compare(d, s, rows, cols, col)) - else: - out["classification_column"] = None - return p, out - - -def cpsr_compare(d, s, rows, cols, col): - """CPSR's own class beside the final class and ClinVar's. - - The final class (CLASSIFICATION) is ClinVar's whenever ClinVar has a - trusted record, so it hides CPSR's own call (CPSR_CLASSIFICATION). This - lists the P/LP rows (final or own class), counts the final P/LP rows that - CPSR's own rules class benign or likely benign, and lists every row where - ClinVar and CPSR's own class disagree on P/LP. ClinVar's class is - CLINVAR_CLASSIFICATION, or the final class where ASSERTION_AUTHORITY is - ClinVar. A table with one class column has no disagreement to show. - Low-confidence: DP_CONTROL below step 6's depth floor (CPSR writes no - GQ). The cDNA read check of step 17 goes on each P/LP row.""" - own = "CPSR_CLASSIFICATION" if "CPSR_CLASSIFICATION" in cols and col != "CPSR_CLASSIFICATION" else None - hits = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - floor = call_floor(hits) if hits else dict(CALL_FLOOR) - cdna = cpsr_cdna_checks(d, s) - norm = clip_check.norm_class - plp, disagree, own_benign = [], [], 0 - for r in rows: - final, mine = norm(r.get(col)), norm(r.get(own)) if own else "" - authority = (r.get("ASSERTION_AUTHORITY") or "").strip() - clinvar_raw = ((r.get("CLINVAR_CLASSIFICATION") or "").strip() if "CLINVAR_CLASSIFICATION" in cols - else (r.get(col) or "").strip() if authority.lower() == "clinvar" else "") - clinvar = norm(clinvar_raw) - if final in clip_check.PLP and mine in ("benign", "likely benign"): - own_benign += 1 - is_plp = final in clip_check.PLP or mine in clip_check.PLP - differ = bool(own and clinvar and mine) and (clinvar in clip_check.PLP) != (mine in clip_check.PLP) - if not (is_plp or differ): - continue - v = clip_check.cpsr_variant(r) - dp = (r.get("DP_CONTROL") or "").strip() - dp = int(dp) if dp.isdigit() else None - x = {"variant": f"{v[0]}:{v[1]} {v[2]}>{v[3]}" if v else (r.get("GENOMIC_CHANGE") or "."), - "gene": r.get("SYMBOL") or ".", "final": r.get(col) or ".", - "cpsr": (r.get(own) or ".") if own else None, "clinvar": clinvar_raw if clinvar else None, - "authority": authority or None, "acmg_codes": (r.get("ACMG_CODE") or "").strip() or None, - "dp": dp, "low_confidence": dp is not None and dp < floor["DP"]} - if cdna is not None and is_plp: - x["cdna"] = (cdna.get((v[0], v[1], v[2], v[3])) if v else None) or { - "check": "not_checked", "note": "not in the read-check table", "alt_reads": None, - "alt_clipped": None, "ref_reads": None, "ref_clipped": None} - if is_plp: - plp.append(x) - if differ: - disagree.append(x) - out = {"plp": plp, "disagreements": disagree, "final_plp_own_benign": own_benign, - "own_class_column": own, "low_confidence": sum(1 for x in plp if x["low_confidence"]), - "call_floor_dp": floor["DP"]} - if cdna is not None: - out["cdna_flagged"] = sum(1 for x in plp if x["cdna"]["check"] == "flagged") - return out - - -def csq_rank(fmt, csq): - """The rule of bin/csq_rank.awk on one CSQ value: (class, entries, freq). - class is MANE (entries of the MANE Select transcript), else CANONICAL, - else ANY (every entry); freq is rare, common, unknown (no value) or none - (the CSQ has no frequency field): MAX_AF, else gnomADe_AF and gnomADg_AF, - the highest over the entries.""" - ix = {k: i for i, k in enumerate(fmt)} - entries = [v for v in (t.split("|") for t in csq.split(",") if t) if len(v) == len(fmt)] - - def get(v, k): - return v[ix[k]] if k in ix else "" - mane = [v for v in entries if get(v, "MANE_SELECT")] - canon = [v for v in entries if get(v, "CANONICAL") == "YES"] - cls, chosen = ("MANE", mane) if mane else ("CANONICAL", canon) if canon else ("ANY", entries) - fields = ["MAX_AF"] if "MAX_AF" in ix else [f for f in ("gnomADe_AF", "gnomADg_AF") if f in ix] - afs = [] - for v in entries: - for f in fields: - for a in get(v, f).split("&"): - try: - afs.append(float(a)) - except ValueError: - pass - freq = "none" if not fields else "unknown" if not afs else "rare" if max(afs) < COMMON_AF else "common" - return cls, [{"impact": get(v, "IMPACT"), "gene": get(v, "SYMBOL"), - "consequence": get(v, "Consequence").replace("&", ",")} for v in chosen], freq - - -def call_check(fmt, sample, floor): - """bin/csq_rank.awk's genotype check of one sample: (weak, reasons, gq, - dp, ad). Weak: GQ or DP below the floor, a heterozygote's alt read - fraction outside HET_AB, a homozygous alt call's below HOM_AB.""" - f = dict(zip(fmt.split(":"), sample.split(":"))) - gq, dp = (int(f[k]) if f.get(k, "").isdigit() else None for k in ("GQ", "DP")) - ad = f.get("AD", "") - why = [] - if gq is not None and gq < floor["GQ"]: - why.append(f"GQ {gq} < {floor['GQ']}") - if dp is not None and dp < floor["DP"]: - why.append(f"DP {dp} < {floor['DP']}") - al = f.get("GT", "").replace("|", "/").split("/") - depths = ad.split(",") - if len(al) == 2 and all(a.isdigit() for a in al) and len(depths) > 1 and all(x.isdigit() for x in depths): - tot = sum(int(x) for x in depths) - if tot: - ab = sum(int(x) for x in depths[1:]) / tot - lo, hi = floor["HET_AB"] - if (al[0] == "0") != (al[1] == "0") and not lo <= ab <= hi: - why.append(f"alt reads {ab:.2f} of the depth on a heterozygote") - elif al[0] == al[1] != "0" and ab < floor["HOM_AB"]: - why.append(f"alt reads {ab:.2f} of the depth on a homozygous call") - return bool(why), why, gq, dp, ad or None - - -def clinical_call_floor(path): - """The floors bin/csq_rank.awk wrote into the clinical VCF, the defaults without the line.""" - floor = dict(CALL_FLOOR, **AB_FLOOR) - try: - with open_text(path) as f: - for line in f: - if not line.startswith("##"): - break - if line.startswith("##PGP_CALL_FLOOR="): - for k, v in re.findall(r"\b(GQ|DP)=(\d+)", line): - floor[k] = int(v) - m = re.search(r"HET_AB=([0-9.]+)-([0-9.]+)", line) - if m: - floor["HET_AB"] = (float(m.group(1)), float(m.group(2))) - m = re.search(r"HOM_AB=([0-9.]+)", line) - if m: - floor["HOM_AB"] = float(m.group(1)) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(path)}: {e}") from e - return floor - - -def csq_high_impact_genes(path): - """{(chrom, pos, ref, alt): record} of the records whose ranked CSQ entries - (bin/csq_rank.awk's rule: the MANE Select transcript, else the canonical - one, else any) hold a HIGH-impact consequence. record: genotype, genes - ({gene: consequence}), transcript (MANE, CANONICAL or ANY), frequency - (rare, common, unknown or none), weak (the call check failed), why, gq, - dp and ad. A HIGH consequence on a transcript other than the MANE Select - one does not count when the record has a MANE Select transcript.""" - floor = clinical_call_floor(path) - fmt, out = None, {} - try: - with open_text(path) as f: - for line in f: - if line.startswith("##INFO= 9 else "" - weak, why, gq, dp, ad = call_check(r[8], r[9], floor) if len(r) > 9 else (False, [], None, None, None) - out[(r[0], int(r[1]), r[3], r[4])] = { - "genotype": gt, "ps": gt_field(r, "PS"), "genes": genes, "transcript": cls, "frequency": freq, - "weak": weak, "why": why, "gq": gq, "dp": dp, "ad": ad} - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(path)}: {e}") from e - return out - - -# --- the ACMG SF rules and the carrier list (assets/) -------------------------- - -_ASSETS = {} - - -def asset_rows(name): - """The rows of assets/ (comment lines left out), read once.""" - p = os.path.join(ASSETS_DIR, name) - if p in _ASSETS: - return _ASSETS[p] - if not os.path.isfile(p): - raise Unreadable(f"{name} is not in {ASSETS_DIR}: the report needs the repository's assets/ folder " - "(render_report.py --assets)") - with open(p, encoding="utf-8") as f: - rows = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) - _ASSETS[p] = rows - return rows - - -def allele_key(locus): - """'chr6:26092913:G:A' -> ('chr6', 26092913, 'G', 'A').""" - c, pos, ref, alt = locus.split(":") - return (c, int(pos), ref, alt) +def allele_key(locus): + """'chr6:26092913:G:A' -> ('chr6', 26092913, 'G', 'A').""" + c, pos, ref, alt = locus.split(":") + return (c, int(pos), ref, alt) def parse_alleles(text, kinds=False): @@ -3072,1056 +334,3 @@ def parse_alleles(text, kinds=False): raise ValueError(f"severity '{kind}' is not one of {', '.join(SEVERITY_KINDS)}: {e}") out[allele_key(parts[0])] = {"kind": kind, "name": parts[1] if len(parts) > 1 else parts[0]} return out - - -def acmg_sf_rules(): - """{gene: rule} of assets/acmg_sf.tsv.""" - rules = {} - for n, r in enumerate(asset_rows("acmg_sf.tsv"), 1): - try: - g = r["gene"] - if r["report_if"] not in SF_REPORT_IF or r["sv_cn"] not in ("yes", "no"): - raise ValueError(f"report_if must be one of {', '.join(SF_REPORT_IF)}, sv_cn yes or no") - if r["sv_cn"] == "yes" and r["report_if"] not in ("any", "het_female_too"): - raise ValueError("sv_cn yes needs a rule that one variant meets (any, het_female_too)") - regions = [] - for e in (r["sv_regions"] or ".").split(";"): - e = e.strip() - if e not in ("", "."): - loc, _, name = e.partition(" ") - c, _, span = loc.partition(":") - a, _, b = span.partition("-") - regions.append((c, int(a), int(b), name or loc)) - variants = parse_alleles(r["variants"]) - if r["report_if"] == "hom_variant" and not variants: - raise ValueError("report_if hom_variant needs its variants") - if g in rules: - raise ValueError("the gene is on two rows") - rules[g] = {"inheritance": r["inheritance"], "report_if": r["report_if"], "variants": variants, - "sv_cn": r["sv_cn"] == "yes", "sv_regions": regions, - "severity": parse_alleles(r["severity"], kinds=True), - "table_text": r["table_text"], "note": None if r["note"] in ("", ".") else r["note"]} - except (KeyError, ValueError, TypeError) as e: - raise Unreadable(f"acmg_sf.tsv row {n}: {e}") from e - if not rules: - raise Unreadable("acmg_sf.tsv has no gene") - return rules - - -def carrier_genes(): - """{gene: row} of assets/carrier_genes.tsv, the severity cell parsed.""" - out = {} - for n, r in enumerate(asset_rows("carrier_genes.tsv"), 1): - try: - out[r["gene"]] = {"inheritance": r["inheritance"], "lists": r["lists"], "condition": r["condition"], - "severity": parse_alleles(r.get("severity"), kinds=True), - "span": (r["chrom"], int(r["start"]), int(r["end"]))} - except (KeyError, ValueError, TypeError) as e: - raise Unreadable(f"carrier_genes.tsv row {n}: {e}") from e - return out - - -def allele_copies(gt): - """(alt copies, ploidy) of a GT: '0/1' (1, 2), '1/1' (2, 2), '1' (1, 1), - '1/.' (1, 2): a missing allele is a copy not called, not a missing copy, - so the half of a split 1/2 call is not read as hemizygous.""" - a = [x for x in re.split(r"[/|]", gt or "") if x != ""] - return sum(1 for x in a if x not in ("0", ".")), len(a) - - -def zygosity_word(copies, ploidy): - if not copies: - return "unknown" - if ploidy == 1: - return "hemizygous" - return "hom" if copies >= 2 else "het" - - -# --- two hits in one gene across callers: a deletion plus a small variant -------- -# -# Small variants alone miss a gene hit twice when one hit is a deletion: an -# exon deletion of one copy of MUTYH plus a pathogenic variant on the other, an -# HBA deletion plus an HBA point variant, one SMN1 copy plus an SMN1 variant, a -# 1q21.1 deletion plus the RBM8A low-expression allele (TAR syndrome: Albers et -# al., Nat Genet 2012;44(4):435-439, PMID 22366785). The deletions come from the -# engines that call them: AnnotSV's class 4 and 5 deletions (step 05), step 35's -# interval table (bin/cn_loci.tsv), its HBA count and Parascopy's SMN1 count. -# Two hits are in trans (biallelic) only when the reads show it: the variant sits -# inside the deletion and is called homozygous (only one copy is left, and it -# carries the variant), or both calls carry a read-backed phase (GT with "|" and -# the same PS) on different copies. Otherwise they are possibly in trans; on the -# same copy (cis) they are no second hit. -TWO_HIT_ORDER = ("biallelic", "possibly biallelic") - - -def gene_deletions(d, s, spans): - """The genes a called deletion reaches: ({gene: [deletion]}, engines read, - engines not read). spans: {gene: (chrom, start, end)}, 1-based inclusive - (assets/carrier_genes.tsv). A deletion: chrom, start, end (1-based, - inclusive), text, inside (a variant inside its span is on the one copy left; - False for the HBA count, which cannot say which gene of the pair is gone), - gt and ps (its phase, AnnotSV only).""" - out, read, unread = {}, [], [] - - def add(g, x): - out.setdefault(g, []).append(x) - - def over(chrom, a, b): - return [g for g, (c, gs, ge) in spans.items() if c == chrom and gs <= b and ge >= a] - try: - p, data = sec_annotsv(d, s) - except (Unreadable, OSError, ValueError, KeyError) as e: - p, data = None, {} - unread.append(f"AnnotSV (step 05): {e}") - if p: - read.append("AnnotSV class 4 and 5 deletions (step 05)") - for x in data["rows"]: - chrom, _, span = x["sv"].partition(":") - a, _, b = span.partition("-") - if x["type"] != "DEL" or x["common"] or not (a.isdigit() and b.isdigit()): - continue - # A call genotyped 0/0 (or 0|0) is no deletion in this sample: it pairs with nothing. - if x.get("gt") and set(x["gt"].replace("|", "/").split("/")) == {"0"}: - continue - chrom = chrom if chrom.startswith("chr") else "chr" + chrom - # SV_start is the base before the deleted bases (annotsv_postfilter.sv_span): - # they are SV_start+1 to SV_end, 1-based. The text keeps AnnotSV's coordinates. - first = int(a) + 1 - hit = set(over(chrom, first, int(b))) - # A listed gene AnnotSV names counts only when its span reaches the deleted bases: one - # that ends at the padding base loses none. A gene with no span here is taken as named. - named = {g for g in x["genes"] if g not in spans} - for g in sorted(hit | named): - add(g, {"chrom": chrom, "start": first, "end": int(b), "inside": True, - "text": f"DEL {chrom}:{a}-{b} (AnnotSV class {x['acmg_class']}, {x['caller']})", - "gt": x.get("gt"), "ps": x.get("ps")}) - elif not unread: - unread.append("AnnotSV (step 05): no table") - try: - p, data = sec_parascopy(d, s) - why = "none" - except (Unreadable, OSError, ValueError, KeyError) as e: - p, data, why = None, {}, str(e) - if not p: - unread.append(f"step 35's copy-number tables: {why}") - return out, read, unread - iv = data.get("cn_intervals") - if iv is not None: - read.append("step 35's interval table (read depth)") - for r in iv["changes"]: - if r["change"] != "deletion" or not (r["start"].isdigit() and r["end"].isdigit()): - continue - a, b = int(r["start"]) + 1, int(r["end"]) # the table is BED: 0-based start - for g in over(r["chrom"], a, b): - add(g, {"chrom": r["chrom"], "start": a, "end": b, "inside": True, "gt": None, "ps": None, - "text": f"{r['locus']} deletion, {r['copies']} of {r['expected']} copies " - "(read depth, step 35)"}) - else: - unread.append("step 35's interval table: none") - hba = data.get("hba") or {} - n = hba.get("alpha_globin_genes", "") - if hba.get("status") == "called" and n.isdigit(): - read.append("step 35's HBA1/HBA2 count") - # 0 genes: no alpha-globin gene is left to carry a variant, so a hit there pairs with nothing. - if 1 <= int(n) < 4: - for g in ("HBA1", "HBA2"): - if g in spans: - c, a, b = spans[g] - add(g, {"chrom": c, "start": a, "end": b, "inside": False, "gt": None, "ps": None, - "text": f"{n} of 4 alpha-globin genes (read depth, step 35)"}) - else: - unread.append(f"step 35's HBA1/HBA2 count: {hba.get('message') or 'not called'}") - smn1 = (data.get("smn1") or {}).get("cn") - if smn1 is not None: - read.append("Parascopy's SMN1 count (step 35)") - if smn1 == 1 and "SMN1" in spans: - c, a, b = spans["SMN1"] - add("SMN1", {"chrom": c, "start": a, "end": b, "inside": True, "gt": None, "ps": None, - "text": "one SMN1 copy (Parascopy, step 35)"}) - else: - unread.append("Parascopy's SMN1 count: not resolved") - return out, read, unread - - -def phase_haps(gt): - """The copies (haplotype indexes) a phased GT puts the alt allele on, or - None when the GT is not phased or a copy is not called: '0|1' -> {1}, - '1|0' -> {0}, '0/1' -> None, '.|1' -> None (the missing copy could carry - the other allele, so no cis or trans can be read from it).""" - if not gt or "|" not in gt: - return None - alleles = gt.split("|") - if "." in alleles: - return None - return frozenset(i for i, a in enumerate(alleles) if a != "0") - - -def pair_phase(dl, v): - """(verdict, why) of a deletion and a small variant in one gene: verdict - biallelic, possibly biallelic, or cis (no second hit).""" - inside = dl["inside"] and dl["chrom"] == v["chrom"] and dl["start"] <= v["pos"] <= dl["end"] - # Read-backed phase first: on the same copy (cis) is no second hit, wherever the variant sits. - hd, hv = phase_haps(dl.get("gt")), phase_haps(v.get("gt")) - phased = None - if hd and hv and dl.get("ps") and dl.get("ps") == v.get("ps"): - if hd == hv: - return "cis", f"in cis: read-backed phase puts both on the same copy (phase set {v['ps']})" - phased = (f"in trans: read-backed phase puts them on different copies (phase set {v['ps']})" if not hd & hv - else f"read-backed phase puts the variant on both copies (phase set {v['ps']}); check the reads") - if inside and v["copies"] >= 2: - return "biallelic", ("the variant is called homozygous inside the deletion: it is hemizygous, on the one " - "copy the deletion leaves") - if inside and v["ploidy"] == 2: - return "possibly biallelic", ("a heterozygous call inside a called deletion: one of the two calls may be " - "wrong; check the reads") - if phased: - return ("biallelic" if phased.startswith("in trans") else "possibly biallelic"), phased - return "possibly biallelic", "phase unknown, possibly in trans (no read-backed phase links the two calls)" - - -def two_hits(variants, dels): - """The genes hit twice: ({gene: entry}, {(gene, key): deletion text} of - the variants called homozygous inside a deletion, shown as hemizygous). - variants: [{gene, key, chrom, pos, text, genotype, copies, ploidy, gt, ps, - allele}] (allele: the severity name of a hypomorph, or None). An entry is - the best pair of the gene: verdict, why, deletion, variant, genotype, and - the allele name; a gene whose every pair is in cis has none. pairs: - {(gene, key): " with : "}, the best pair of each - variant that has one.""" - entries, hemi, pairs = {}, {}, {} - for v in variants: - for dl in dels.get(v["gene"], []): - verdict, why = pair_phase(dl, v) - if verdict == "biallelic" and why.startswith("the variant is called homozygous"): - hemi[(v["gene"], v["key"])] = dl["text"] - if verdict == "cis": - continue - e = {"gene": v["gene"], "verdict": verdict, "why": why, "deletion": dl["text"], "variant": v["text"], - "genotype": "hemizygous" if (v["gene"], v["key"]) in hemi else v["genotype"], - "allele": v.get("allele")} - old = entries.get(v["gene"]) - if old is None or TWO_HIT_ORDER.index(verdict) < TWO_HIT_ORDER.index(old["verdict"]): - entries[v["gene"]] = e - k = (v["gene"], v["key"]) - if k not in pairs or TWO_HIT_ORDER.index(verdict) < TWO_HIT_ORDER.index(pairs[k][0]): - pairs[k] = (verdict, f"{verdict} with {dl['text']}: {why}") - return entries, hemi, {k: t for k, (_, t) in pairs.items()} - - -def sf_verdicts(gene, rule, alleles): - """Which of a gene's variants the ACMG rule reports. alleles: {key: - (copies, ploidy)} of the gene's distinct small variants, structural ones - keyed by their span. {key: None (reported) or the reason it is not}, and - the partial-deficiency note of a reported gene with a hypomorph allele.""" - sev = rule["severity"] - out, kept = {}, {} - for k, cp in alleles.items(): - if sev.get(k, {}).get("kind") == "pseudodeficiency": - out[k] = f"{sev[k]['name']}: a pseudodeficiency allele, not counted" - else: - kept[k] = cp - n = sum(c for c, _ in kept.values()) - ri = rule["report_if"] - if ri in ("any", "het_female_too"): - ok = {k: True for k in kept} - elif ri == "biallelic": - ok = {k: n >= 2 for k in kept} - elif ri == "hemizygous": - hemi = any(p == 1 for _, p in kept.values()) - ok = {k: hemi or n >= 2 for k in kept} - else: # hom_variant - ok = {k: k in rule["variants"] and c >= 2 and p == 2 for k, (c, p) in kept.items()} - for k, good in ok.items(): - if good: - out[k] = None - elif ri == "hom_variant": - out[k] = f"{gene} is reported only for: {rule['table_text']}" - elif ri == "hemizygous": - out[k] = f"one heterozygous variant; {gene} is reported for {rule['table_text']}" - else: - out[k] = f"one heterozygous variant; {gene} is reported with {rule['table_text']}" - partial = None - hypo = [sev[k]["name"] for k, why in out.items() if why is None and sev.get(k, {}).get("kind") == "hypomorph"] - if hypo: - partial = (f"partial, not profound: {'; '.join(hypo)} is a hypomorph allele, so the deficiency it causes " - "is partial") - return out, partial - - -def sf_structural(d, s, rules): - """Step 05's class 4 and 5 SVs (AnnotSV) in genes whose rule allows them - (sv_cn yes), and the class 4 or 5 deletions over a gene's sv_regions (the - 3' end of EPCAM for MSH2). Common SVs are left out. (rows, checked, note).""" - try: - p, data = sec_annotsv(d, s) - except (Unreadable, OSError, ValueError, KeyError) as e: - return [], False, f"the AnnotSV table could not be read: {e}" - if not p: - return [], False, None - out = [] - for x in data["rows"]: - if x["common"] or str(x["acmg_class"]) not in ("4", "5"): - continue - chrom, _, span = x["sv"].partition(":") - a, _, b = span.partition("-") - chrom = chrom if chrom.startswith("chr") else "chr" + chrom - hits = {} - for g in x["genes"]: - if g in rules and rules[g]["sv_cn"]: - hits[g] = None - for g, rule in rules.items(): - for c, rs, re_, name in rule["sv_regions"]: - if (x["type"] == "DEL" and c == chrom and a.isdigit() and b.isdigit() - and int(a) <= re_ and int(b) >= rs and g not in hits): - hits[g] = name - for g, via in sorted(hits.items()): - out.append({"gene": g, "variant": f"{x['type']} {chrom}:{a}-{b}", "acmg_class": str(x["acmg_class"]), - "caller": x["caller"], "support": x["support"], "via": via, - "key": ("sv", chrom, a, b, x["type"])}) - return out, True, None - - -def acmg_sf_tier(d, s, clinical_vcf): - """The variants that are ACMG secondary findings by the rule of their gene - (assets/acmg_sf.tsv): the ClinVar P/LP hits of step 6, the clinical - filter's rare HIGH-impact records, and step 05's class 4 and 5 SVs in the - genes whose rule allows them. A variant in an SF gene that its rule does - not report (one MUTYH variant, a heterozygous HFE p.C282Y) is listed - under not_reported with the reason.""" - rules = acmg_sf_rules() - clinvar, high = [], [] - hits = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - if hits: - floor = call_floor(hits) - for r in vcf_records(hits): - if len(r) < 8: - continue - row = clinvar_row(r, floor) - copies, ploidy = allele_copies(r[9].split(":")[0] if len(r) > 9 else "") - for g in row["gene"].split(","): - if g in rules: - clinvar.append({"gene": g, "variant": f"{row['chrom']}:{row['pos']} {row['ref']}>{row['alt']}", - "genotype": row["genotype"], "significance": row["significance"], - "stars": row["stars"], "low_confidence": row["low_confidence"], - "key": (row["chrom"], row["pos"], row["ref"], row["alt"]), - "copies": copies, "ploidy": ploidy, - "gt": r[9].split(":")[0] if len(r) > 9 else "", "ps": gt_field(r, "PS")}) - if clinical_vcf: - # A common record is in the clinical VCF through the ClinVar tier, - # not as a rare HIGH impact; one of unknown frequency is listed and - # marked, never counted as rare; a weak call is listed and marked. - for (c, pos, ref, alt), x in sorted(csq_high_impact_genes(clinical_vcf).items()): - if x["frequency"] == "common": - continue - for g, cons in sorted(x["genes"].items()): - if g in rules: - gt = x["genotype"] - copies, ploidy = allele_copies(gt) - zyg = {"0/1": "het", "1/0": "het", "0|1": "het", "1|0": "het", "1/1": "hom", "1|1": "hom"}.get(gt, gt) - high.append({"gene": g, "variant": f"{c}:{pos} {ref}>{alt}", "genotype": zyg, "consequence": cons, - "transcript": x["transcript"], "frequency": x["frequency"], - "frequency_unknown": x["frequency"] == "unknown", - "weak_call": x["weak"], "call_note": "; ".join(x["why"]) or None, - "gq": x["gq"], "dp": x["dp"], "ad": x["ad"], - "key": (c, pos, ref, alt), "copies": copies, "ploidy": ploidy, - "gt": gt, "ps": x.get("ps")}) - svs, sv_checked, sv_note = sf_structural(d, s, rules) - # One verdict per gene, over its distinct variants: a ClinVar hit that is - # also a HIGH-impact record is one variant. - alleles = {} - for x in clinvar + high: - alleles.setdefault(x["gene"], {}).setdefault(x["key"], (max(x["copies"], 1), x["ploidy"] or 2)) - for x in svs: - alleles.setdefault(x["gene"], {}).setdefault(x["key"], (1, 2)) - # A called deletion of a gene whose rule needs two variants counts as one - # of them (two_hits), unless read-backed phase puts it on the variant's copy. - two_hit_note = None - try: - spans = {g: x["span"] for g, x in carrier_genes().items()} - except Unreadable as e: - spans = {} - two_hit_note = (f"A deletion as the second hit: only AnnotSV's gene names were read; step 35's deletions " - f"need the gene spans of {e}") - dels, _, _ = gene_deletions(d, s, spans) - seen, variants = set(), [] - for x in clinvar + high: - pseudo = rules[x["gene"]]["severity"].get(x["key"], {}).get("kind") == "pseudodeficiency" - if (x["gene"], x["key"]) not in seen and not pseudo: - seen.add((x["gene"], x["key"])) - variants.append({"gene": x["gene"], "key": x["key"], "chrom": x["key"][0], "pos": x["key"][1], - "text": x["variant"], "genotype": x["genotype"], "copies": x["copies"], - "ploidy": x["ploidy"] or 2, "gt": x["gt"], "ps": x["ps"], "allele": None}) - entries, hemi, pairs = two_hits(variants, dels) - two = {} - for g in entries: - if rules[g]["report_if"] in ("biallelic", "hemizygous"): - alleles[g][("deletion", g)] = (1, 2) - two.update({k: t for k, t in pairs.items() if k[0] == g}) - verdict = {g: sf_verdicts(g, rules[g], a) for g, a in alleles.items()} - reported = {"clinvar": [], "high": [], "sv": []} - not_reported = [] - for kind, rows, source in (("clinvar", clinvar, "ClinVar"), ("high", high, "HIGH impact"), ("sv", svs, "AnnotSV")): - for x in rows: - why, partial = verdict[x["gene"]][0][x["key"]], verdict[x["gene"]][1] - rule = rules[x["gene"]] - key = x["key"] - if (x["gene"], key) in hemi: - x["genotype"] = "hemizygous" - for k in ("key", "copies", "ploidy", "gt", "ps"): - x.pop(k, None) - if why is None: - x.update(rule=rule["table_text"], rule_note=rule["note"], severity=partial) - if kind != "sv" and (x["gene"], key) in two: - x["two_hit"] = two[(x["gene"], key)] - reported[kind].append(x) - else: - not_reported.append({"gene": x["gene"], "variant": x["variant"], "genotype": x.get("genotype"), - "source": source, "reason": why}) - high = reported["high"] - sure = [x for x in high if not (x["weak_call"] or x["frequency_unknown"])] - return {"version": ACMG_SF_VERSION, "genes_on_list": len(rules), "rules": "assets/acmg_sf.tsv", - "clinvar_hits": reported["clinvar"], "clinvar_checked": bool(hits), - "high_impact": high, "high_impact_checked": bool(clinical_vcf), - "high_impact_rare": len(sure), - "high_impact_weak": sum(1 for x in high if x["weak_call"]), - "high_impact_frequency_unknown": sum(1 for x in high if x["frequency_unknown"]), - "structural": reported["sv"], "structural_checked": sv_checked, "structural_note": sv_note, - "not_reported": not_reported, "two_hit_note": two_hit_note} - - -def gt_field(r, key): - """One FORMAT field of a VCF record's first sample, or None.""" - if len(r) < 10: - return None - v = dict(zip(r[8].split(":"), r[9].split(":"))).get(key) - return None if v in (None, "", ".") else v - - -def carrier_list(p, d=None, s=None): - """The sample's own carrier list: the ClinVar hits of step 6 (p) in the - genes of assets/carrier_genes.tsv, one person, no partner needed. An - allele the list marks hypomorph or pseudodeficiency is set aside, never a - carrier finding on its own. With the sample folder (d, s), the genes of - autosomal recessive conditions that a called deletion and a hit both reach - (two_hits), and a hit called homozygous inside a deletion is shown as - hemizygous.""" - genes = carrier_genes() - carriers, aside, variants = [], [], [] - for r in vcf_records(p): - if len(r) < 8: - continue - row = clinvar_row(r) - gt = r[9].split(":")[0] if len(r) > 9 else "" - copies, ploidy = allele_copies(gt) - key = (row["chrom"], row["pos"], row["ref"], row["alt"]) - for g in row["gene"].split(","): - if g not in genes: - continue - x = genes[g] - item = {"gene": g, "variant": f"{row['chrom']}:{row['pos']} {row['ref']}>{row['alt']}", - "genotype": row["genotype"] if half_call(gt) else zygosity_word(copies, ploidy), - "significance": row["significance"], - "stars": row["stars"], "inheritance": x["inheritance"], "condition": x["condition"]} - sev = x["severity"].get(key) - if sev: - item.update(kind=sev["kind"], name=sev["name"]) - aside.append(item) - else: - carriers.append(item) - if not sev or sev["kind"] == "hypomorph": - variants.append({"gene": g, "key": key, "chrom": row["chrom"], "pos": row["pos"], - "text": item["variant"], "genotype": item["genotype"], "copies": copies, - "ploidy": ploidy, "gt": gt, "ps": gt_field(r, "PS"), - "allele": sev["name"] if sev else None, "item": item}) - out = {"genes_on_list": len(genes), "list": "assets/carrier_genes.tsv", - "carriers": carriers, "set_aside": aside} - if d is not None: - dels, read, unread = gene_deletions(d, s, {g: x["span"] for g, x in genes.items()}) - entries, hemi, _ = two_hits(variants, dels) - for v in variants: - if (v["gene"], v["key"]) in hemi: - v["item"].update(genotype="hemizygous", - hemizygous=f"called homozygous inside {hemi[(v['gene'], v['key'])]}") - rows = [] - for g, e in sorted(entries.items()): - if genes[g]["inheritance"] == "AR": - e.update(condition=genes[g]["condition"], inheritance="AR") - rows.append(e) - out["two_hits"] = {"genes": rows, "deletions_read": read, "deletions_not_read": unread} - return out - - -def acmg_sf_or_note(d, s, clinical_vcf): - """acmg_sf_tier, or the reason it could not run (a missing or broken - assets/acmg_sf.tsv), which both reports print: the clinical counts stay.""" - try: - return acmg_sf_tier(d, s, clinical_vcf) - except Unreadable as e: - return {"version": ACMG_SF_VERSION, "unavailable": str(e), "clinvar_hits": [], "high_impact": [], - "structural": [], "not_reported": []} - - -def carrier_list_or_note(p, d=None, s=None): - """carrier_list, or the reason it could not run, which both reports print: - the ClinVar hits stay.""" - try: - return carrier_list(p, d, s) - except Unreadable as e: - return {"unavailable": str(e)} - - -def sec_clinical(d, s): - p = first_existing(d, [f"clinical/{s}_clinical_summary.tsv"]) - v = first_existing(d, [f"clinical/{s}_clinical.vcf.gz"]) - if not p: - # The Nextflow report gets the clinical VCF, not the summary table: - # the count only (one record per variant, as in the table). - if not v: - return None, {} - return v, {"variants": sum(1 for _ in vcf_records(v)), "acmg_sf": acmg_sf_or_note(d, s, v)} - rows = read_tsv(p) - by = {} - for r in rows: - by[r.get("IMPACT") or "."] = by.get(r.get("IMPACT") or ".", 0) + 1 - genes = sorted({r.get("GENE") for r in rows if r.get("GENE") not in (None, "", ".")}) - out = {"variants": len(rows), "by_impact": by, "genes": len(genes), "acmg_sf": acmg_sf_or_note(d, s, v)} - # Step 23's tier and call check columns; a table written before them has neither. - if rows and "FREQ_TIER" in rows[0]: - out["frequency_unknown"] = sum(1 for r in rows if r.get("FREQ_TIER") == "unknown") - if rows and "CALL_CHECK" in rows[0]: - out["weak_calls"] = sum(1 for r in rows if r.get("CALL_CHECK") == "weak") - return p, out - - -def sec_slivar(d, s): - p = first_existing(d, [f"slivar/{s}_slivar_summary.tsv"]) - if not p: - # The Nextflow report gets the prioritized VCF: the count, plus the - # phase when COMPHET_PHASE's pairs table sits beside it (HTML_REPORT - # stages it there). - v = first_existing(d, [f"slivar/{s}_prioritized.vcf.gz"]) - if not v: - return None, {} - out = {"prioritized": sum(1 for _ in vcf_records(v))} - out.update(comphet_phase(os.path.join(os.path.dirname(v), f"{s}_compound_het_pairs.tsv"))) - return v, out - rows = read_tsv(p) - out = {"prioritized": len(rows)} - # Step 31's frequency tier: prioritized for the compound-het search, not rare. - if rows and "FREQ_TIER" in rows[0]: - out["frequency_unknown"] = sum(1 for r in rows if r.get("FREQ_TIER") == "unknown") - ch = os.path.join(os.path.dirname(p), f"{s}_compound_hets.tsv") - if os.path.isfile(ch): - crow = read_tsv(ch) - out["compound_het_variants"] = len(crow) - out["compound_het_genes"] = len({r.get("GENE") for r in crow if r.get("GENE")}) - out.update(comphet_phase(os.path.join(os.path.dirname(p), f"{s}_compound_het_pairs.tsv"))) - return p, out - - -# The trans pairs the report lists by name; the rest are counted. -COMPHET_TRANS_SHOWN = 20 - - -def comphet_phase(path): - """Step 31's pairs table (bin/comphet_phase.py): the pairs by phase from - the reads, and the trans ones (both copies of the gene hit) by name. A run - from before the table has none of these keys.""" - if not os.path.isfile(path): - return {} - rows = read_tsv(path) - phase = {k: sum(1 for r in rows if r.get("PHASE") == k) for k in ("trans", "cis", "unknown")} - - def n(x): - return int(x) if (x or "").isdigit() else 0 - trans = [{"gene": r.get("GENE", ""), - "variants": f"{r.get('CHROM')}:{r.get('POS_1')} {r.get('REF_1')}>{r.get('ALT_1')} + " - f"{r.get('CHROM')}:{r.get('POS_2')} {r.get('REF_2')}>{r.get('ALT_2')}", - "reads": f"{r.get('TRANS_READS')} of {n(r.get('TRANS_READS')) + n(r.get('CIS_READS'))}"} - for r in rows if r.get("PHASE") == "trans"] - return {"compound_het_pairs": len(rows), "compound_het_phase": phase, - "compound_het_trans": trans[:COMPHET_TRANS_SHOWN]} - - -def sec_coverage(d, s): - p = first_existing(d, [f"mosdepth/{s}.mosdepth.summary.txt", f"coverage/{s}.mosdepth.summary.txt"]) - if not p: - return None, {} - mean = None - for r in read_tsv(p): - if r.get("chrom") == "total": - mean = r.get("mean") - return p, {"mean_depth": float(mean) if mean not in (None, "") else None} - - -# A gene of assets/callability_genes.tsv counts as read when at least this -# share of its bases has the depth step 16b counts (its thresholds column, -# 10x of reads with MAPQ 20 or more). -CALLABLE_SHARE = 0.9 - - -def callability(catalog, table): - """The reading of every gene of the catalog rows from step 16b's per-gene - table (mosdepth --thresholds over the catalog's genes: chrom, start, end, - gene, bases at the threshold), as the section's data.""" - with open_text(table) as f: - lines = [x.rstrip("\n").split("\t") for x in f if x.strip()] - if not lines or not lines[0][0].startswith("#") or len(lines[0]) != 5 \ - or not re.fullmatch(r"[0-9]+X", lines[0][4]): - raise Unreadable(f"{os.path.basename(table)} is not mosdepth's thresholds table of one depth") - min_depth = int(lines[0][4][:-1]) - bases = {} - for c in lines[1:]: - if len(c) != 5: - raise Unreadable(f"{os.path.basename(table)}: a row has {len(c)} columns, not 5") - bases[c[3]] = (int(c[2]) - int(c[1]), int(c[4])) - if not bases: - raise Unreadable(f"{os.path.basename(table)} has no gene rows") - # mosdepth writes no row on a contig the BAM lacks: a chromosome with no row at all is - # not on the sample's reference; a gene missing from a chromosome that has rows means - # the table was written from another catalog. - contigs = {c[0] for c in lines[1:]} - genes, unowned, owned, not_on_reference = [], [], [], [] - for r in catalog: - g = {"gene": r["gene"], "hgnc_id": r["hgnc_id"], "reasons": r["reasons"].split(","), - "owner": None if r["owner"] == "." else r["owner"], "share": None} - if r["start"] == "." or r["chrom"] not in contigs: - g["status"] = "not on this reference" - not_on_reference.append(g["gene"]) - else: - if r["gene"] not in bases: - raise Unreadable(f"{os.path.basename(table)} has no row for {r['gene']}: it was written from " - "another assets/callability_genes.tsv; run step 16b again") - n, ok = bases[r["gene"]] - g["share"] = round(ok / n, 4) if n else 0.0 - g["status"] = "read" if g["share"] >= CALLABLE_SHARE else "not read" - genes.append(g) - if g["status"] != "read": - (owned if g["owner"] else unowned).append(g) - return {"min_depth": min_depth, "min_mapq": 20, "min_share": CALLABLE_SHARE, "catalog_genes": len(genes), - "read": sum(1 for g in genes if g["status"] == "read"), - "not_read": [{k: g[k] for k in ("gene", "hgnc_id", "reasons", "share", "status")} for g in unowned], - "not_read_owned": [{k: g[k] for k in ("gene", "owner", "share", "status")} for g in owned], - "not_on_reference": not_on_reference} - - -def sec_callability(d, s): - """Step 16b's gene callability (the MOSDEPTH task's in a Nextflow folder).""" - p = first_existing(d, [f"mosdepth/{s}_callability.bed.gz", f"coverage/{s}_callability.bed.gz"]) - if not p: - return None, {} - return p, callability(asset_rows("callability_genes.tsv"), p) - - -# How far a chrX or chrY copy number may sit from a whole number and still be -# read as that number. Further off (a mosaic, a mixed sample, a noisy run) -# names no karyotype. -KARYOTYPE_TOL = 0.3 - - -# The sex-chromosome aneuploidies named, by (chrX, chrY) copies. Any other pair of -# whole numbers (chrY 4, chrX 6, a degenerate 100000) names nothing: in a run it is -# far more often depth noise or a broken input than a karyotype. -KARYOTYPES = {(1, 0): "45,X", (2, 1): "47,XXY", (1, 2): "47,XYY", (3, 0): "47,XXX", - (2, 2): "48,XXYY", (3, 1): "48,XXXY", (4, 0): "48,XXXX", - (4, 1): "49,XXXXY", (5, 0): "49,XXXXX"} - - -def karyotype(cn_x, cn_y, tol=KARYOTYPE_TOL): - """The sex-chromosome karyotype the two copy numbers name, one of - KARYOTYPES; None for 46,XX and 46,XY, for a missing value, for a value - between whole numbers, and for any pair of copies not in KARYOTYPES.""" - x, y = num(cn_x), num(cn_y) - if x is None or y is None or not (-1 < x < 10 and -1 < y < 10): # also keeps inf from round() - return None - rx, ry = int(round(x)), int(round(y)) - if abs(x - rx) > tol or abs(y - ry) > tol: - return None - return KARYOTYPES.get((rx, ry)) - - -def sec_sex_check(d, s): - p = first_existing(d, ["indexcov/indexcov-indexcov.ped", "indexcov/*-indexcov.ped"]) - if not p: - return None, {} - with open(p, errors="replace") as f: - lines = [l.rstrip("\n").split("\t") for l in f if l.strip()] - head = [h.lstrip("#") for h in lines[0]] if lines else [] - if "sex" not in head or len(lines) < 2: - raise Unreadable(f"{os.path.basename(p)} has no sex column or no sample row") - row = dict(zip(head, lines[-1])) - sex = {"1": "male", "2": "female"}.get(row.get("sex", ""), "unknown") - return p, {"inferred_sex": sex, "cn_chrX": row.get("CNchrX", ""), "cn_chrY": row.get("CNchrY", ""), - "karyotype": karyotype(row.get("CNchrX"), row.get("CNchrY"))} - - -# --- sample identity and contamination (step 33) ---------------------------------- - -def num(x): - try: - v = float(x) - except (TypeError, ValueError): - return None - return v if v == v else None # NaN is no value - - -def sec_sample_qc(d, s): - p = first_existing(d, [f"qc/{s}_sample_qc.tsv"]) - if not p: - return None, {} - data = {} - for r in read_tsv(p): - if r.get("key"): - data[r["key"]] = r.get("value", "") - if "inferred_sex" not in data: - raise Unreadable(f"{os.path.basename(p)} has no inferred_sex row") - # somalier's chrX and chrY depth over the autosomal depth: one copy reads 0.5. - # Named only from more than 10 chrX sites, the floor somalier's own sex - # call needs (og qc sample); a thinner ratio is noise. - xr, yr = num(data.get("x_depth_ratio")), num(data.get("y_depth_ratio")) - xs = num(data.get("x_sites")) - data["karyotype"] = (karyotype(2 * xr, 2 * yr) - if xr is not None and yr is not None and xs is not None and math.isfinite(xs) and xs > 10 - else None) - return p, data - - -# --- mosaic loss of chrY or of one chrX (step 33, og qc sex-loss) ------------------- - -MOSAIC_CALLS = ("detected", "not_detected", "gain", "not_estimated") -MOSAIC_NUMBERS = ("depth_ratio", "depth_ratio_expected", "loss_depth", "loss_depth_low", "loss_depth_high", - "loss_af", "loss_af_low", "loss_af_high", "floor") - - -def sec_mosaic_sex_loss(d, s): - """The mosaic_* rows of the step 33 table, without the prefix; none in a - table written before the estimate existed.""" - p = first_existing(d, [f"qc/{s}_sample_qc.tsv"]) - if not p: - return None, {} - data = {r["key"][len("mosaic_"):]: r.get("value") or "" for r in read_tsv(p) - if (r.get("key") or "").startswith("mosaic_")} - if not data: - return None, {} - if data.get("call") not in MOSAIC_CALLS: - raise Unreadable(f"{os.path.basename(p)} has no mosaic_call row ({', '.join(MOSAIC_CALLS)})") - for k in MOSAIC_NUMBERS: - v = num(data.get(k)) - data[k] = v if v is not None and abs(v) != float("inf") else None # inf is no value either - data["x_het_sites"] = int(data["x_het_sites"]) if (data.get("x_het_sites") or "").isdigit() else None - return p, data - - -def sec_read_count(d, s): - """Step 02's read count (and the ALIGN_MINIMAP2 task's): the reads that - went into the aligner and the primary records of the BAM it wrote.""" - p = first_existing(d, [f"qc/{s}_read_count.tsv"]) - if not p: - return None, {} - data = {} - for r in read_tsv(p): - if r.get("key"): - data[r["key"]] = r.get("value", "") - for k in ("input_reads", "primary_records", "missing_reads"): - if not re.fullmatch(r"-?[0-9]+", data.get(k, "")): - raise Unreadable(f"{os.path.basename(p)} has no whole number for {k}") - data[k] = int(data[k]) - if data.get("read_count_check") not in ("ok", "mismatch"): - raise Unreadable(f"{os.path.basename(p)} has no read_count_check row (ok or mismatch)") - return p, data - - -# Sections that are not a step of their own, or not one of one person (the -# couple screen needs two): no "not run" line without them. -NOT_A_STEP = {"read_count", "couple", "carrier"} - -SECTIONS = [ - # key, title, step, reader - ("coverage", "Coverage (mosdepth)", "16b", sec_coverage), - ("callability", "Gene callability (mosdepth)", "16b", sec_callability), - ("sex_check", "Sex check (indexcov)", "16", sec_sex_check), - ("sample_qc", "Sample identity and contamination (somalier, VerifyBamID2)", "33", sec_sample_qc), - ("mosaic_sex_loss", "Mosaic loss of chromosome Y or X", "33", sec_mosaic_sex_loss), - ("read_count", "Reads kept by alignment", "02", sec_read_count), - ("variants", "Variant calling", "03", sec_variants), - ("clinvar", "ClinVar screen", "06", sec_clinvar), - ("couple", "Couple carrier screen", "39", sec_couple), - ("carrier", "Carrier list", "06", sec_carrier), - ("pharmcat", "PharmCAT", "07", sec_pharmcat), - ("cpic", "CPIC drug-gene recommendations", "27", sec_cpic), - ("pypgx", "pypgx", "32", sec_pypgx), - ("cyrius", "Cyrius (CYP2D6)", "21", sec_cyrius), - ("parascopy", "SMN1/SMN2 copy number (Parascopy, short reads)", "35", sec_parascopy), - ("paralogs", "Paralog genes (Paraphase, PacBio HiFi)", "35b", sec_paralogs), - ("hla", "HLA typing (T1K)", "08", sec_hla), - ("prs", "Polygenic risk scores", "25", sec_prs), - ("ancestry_fine", "Fine-grained ancestry (ADMIXTURE, RFMix)", "26b", sec_ancestry_fine), - ("manta", "Structural variants (Manta)", "04", sec_manta), - ("delly", "Structural variants (Delly)", "19", sec_delly), - ("cnvpytor", "Copy number (CNVpytor)", "18", sec_cnvpytor), - ("sv_consensus", "SV consensus merge", "22", sec_sv_consensus), - ("annotsv", "Structural variants, class 4 and 5 (AnnotSV)", "05", sec_annotsv), - ("expansions", "Repeat expansions (ExpansionHunter)", "09", sec_expansions), - ("telomere", "Telomere content (TelomereHunter)", "10", sec_telomere), - ("roh", "Runs of homozygosity", "11", sec_roh), - ("haplogroup", "Mitochondrial haplogroup", "12", sec_haplogroup), - ("y_haplogroup", "Y-chromosome haplogroup (Yleaf)", "37", sec_y_haplogroup), - ("y_tree", "Y placement on the current public tree", "37", sec_y_tree), - ("y_str", "Y-STR markers (HipSTR)", "45", sec_y_str), - ("consumer_export", "Raw-data export (23andMe v5 layout)", "40", sec_consumer_export), - ("bloodtype", "Blood groups (predicted)", "41", sec_bloodtype), - ("lpa", "Lp(a): LPA KIV-2 repeat count", "46", sec_lpa), - ("vntyper", "MUC1 VNTR frameshifts (VNtyper)", "52", sec_vntyper), - ("locityper", "Locityper second opinion at hard loci", "53", sec_locityper), - ("mito", "Mitochondrial variants", "20", sec_mito), - ("traits", "Traits panel", "42", sec_traits), - ("archaic", "Archaic ancestry (IBDmix)", "43", sec_archaic), - ("ancient", "Ancient populations (AADR PCA)", "44", sec_ancient), - ("viral", "Viral DNA in the unmapped reads", "47", sec_viral), - ("exomiser", "Symptom-driven ranking (Exomiser)", "48", sec_exomiser), - ("chip", "Clonal haematopoiesis (CHIP)", "49", sec_chip), - ("mei", "Mobile element insertions (MEGAnE)", "50", sec_mei), - ("mosaic", "Mosaic chromosomal alterations (MoChA)", "51", sec_mocha), - ("cpsr", "Cancer predisposition (CPSR)", "17", sec_cpsr), - ("clinical", "Clinical variant filter", "23", sec_clinical), - ("slivar", "Variant prioritization (slivar)", "31", sec_slivar), -] - - -# --- run status and manifest ----------------------------------------------------- - -def read_run_status(d): - p = os.path.join(d, "logs", "run_status.tsv") - if not os.path.isfile(p): - return None - st = {"started_epoch": None, "started_utc": None, "declared_sex": None, "steps": {}} - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 3 or c[0].startswith("#"): - continue - if c[0] == "meta" and c[1] == "started_epoch": - try: - st["started_epoch"] = float(c[2]) - except ValueError: - pass - elif c[0] == "meta": - st[c[1]] = c[2] - elif c[0] == "step": - st["steps"][c[1]] = c[2] - if st["started_epoch"] is not None: - st["started_utc"] = iso(st["started_epoch"]) - return st - - -def read_manifest(d): - p = os.path.join(d, "run_manifest.tsv") - if not os.path.isfile(p): - return None - rows = [] - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 3 and not c[0].startswith("#"): - rows.append([c[0], c[1], "\t".join(c[2:])]) - return rows - - -def manifest_value(rows, section, key): - for r in rows or []: - if r[0] == section and r[1] == key: - return r[2] - return None - - -def read_consensus(d, s, sections): - """Step 36's CYP2D6 row (pgx_consensus/_pgx_consensus.tsv), or None - when the table is missing or older than a caller's result it was made from.""" - p = first_existing(d, [f"pgx_consensus/{s}_pgx_consensus.tsv"]) - if not p: - return None - for k in ("pypgx", "cyrius"): - src = sections[k]["source"] - if src and os.path.getmtime(os.path.join(d, src)) > os.path.getmtime(p): - return None - row = next((r for r in read_tsv(p) if (r.get("Gene") or "").strip() == "CYP2D6"), None) - if row is None: - return None - return {"result": row.get("Result") or "", "passed_to_pharmcat": row.get("Outside_call") == "yes", - "reason": row.get("Reason") or "", "source": os.path.relpath(p, d)} - - -def cyp2d6_block(sample, sample_dir, sections): - """CYP2D6 from each caller, side by side, judged by step 36's rule - (bin/pgx_outside_calls.py): a no-call in any case (Indeterminate, None, - '.'...) is not a call, and Cyrius counts only with Filter PASS. PharmCAT - calls no CYP2D6 from a VCF: its column is what step 36 passed to it, so - agreement is between pypgx and Cyrius. With step 36's table its verdict - is kept as well.""" - pc = next((g for g in sections["cpic"]["data"].get("genes", []) if g["gene"] == "CYP2D6"), None) - pg = sections["pypgx"]["data"].get("cyp2d6") if sections["pypgx"]["state"] == "ok" else None - cy = sections["cyrius"]["data"] if sections["cyrius"]["state"] == "ok" else None - # A caller whose step failed says so, not 'not run'. - failed = {k: f"failed (step {sections[k]['step']})" for k in ("cpic", "pypgx", "cyrius") - if sections[k]["state"] == "failed"} - pg_ok = pg is not None and pgx_outside_calls.is_call(pg) - cy_ok = cy is not None and pgx_outside_calls.is_call(cy.get("genotype")) and cy.get("filter") == "PASS" - cy_txt = None - if cy is not None: - cy_txt = cy.get("genotype") or "none" - if pgx_outside_calls.is_call(cy_txt) and not cy_ok: - cy_txt += f" (not usable: Filter {cy.get('filter') or 'missing'})" - calls = { - "PharmCAT": (pc["diplotype"] if pc["status"] not in ("not called", "ambiguous") else pc["status"]) if pc - else failed.get("cpic"), - "pypgx": failed.get("pypgx", pg), - "Cyrius": failed.get("cyrius", cy_txt), - } - agree = pgx_outside_calls.same_diplotype(pg, cy["genotype"]) if pg_ok and cy_ok else None - try: - consensus = read_consensus(sample_dir, sample, sections) - except (OSError, csv.Error) as e: - consensus = {"result": "unreadable", "passed_to_pharmcat": False, "reason": str(e), "source": None} - # pypgx's call alone, labelled, when step 36's rule allows it - # (pgx_outside_calls.probable_cyp2d6); it never reaches PharmCAT. - probable = None - if pg_ok: - pd = sections["pypgx"]["data"] - ev = pd.get("cyp2d6_evidence") or {} - probable = pgx_outside_calls.probable_cyp2d6( - {"Diplotype": pg, "Phenotype": ev.get("phenotype"), "Defining_filter": ev.get("defining_filter"), - "Quality": ev.get("quality")}, - (cy.get("genotype"), cy.get("filter")) if cy is not None else None, - (pd.get("depth_check") or {}).get("status")) - return {"calls": calls, "agree": agree, "consensus": consensus, "probable": probable} - - -def collect(sample, sample_dir, declared_sex=None): - status = read_run_status(sample_dir) - manifest = read_manifest(sample_dir) - sections = {} - for key, title, step, reader in SECTIONS: - sec = {"title": title, "step": step, "state": "missing", "source": None, - "file_date": None, "note": None, "data": {}} - try: - path, data = reader(sample_dir, sample) - except Unreadable as e: - sec["state"], sec["note"] = "unreadable", str(e) - sections[key] = sec - continue - except (OSError, ValueError, KeyError, IndexError, EOFError, csv.Error, zlib.error) as e: - sec["state"], sec["note"] = "unreadable", f"{type(e).__name__}: {e}" - sections[key] = sec - continue - if not path and status and (status["steps"].get(step) or "").startswith("failed"): - # run-all.sh records 'failed' for a step it ran that exited non-zero - sec["state"] = "failed" - sec["note"] = (f"step {step} failed in the run of {status['started_utc'] or 'unknown date'}; " - "its log is in logs/") - if path: - mtime = os.path.getmtime(path) - sec.update(state="ok", source=os.path.relpath(path, sample_dir), file_date=iso(mtime), data=data) - if status and status["started_epoch"] is not None and step in RUN_ALL_STEPS: - result = status["steps"].get(step) - if result != "ok" and mtime < status["started_epoch"]: - sec["state"] = "stale" - sec["note"] = (f"from an earlier run ({iso(mtime)}); step {step} was " - f"{result or 'not run'} in the run of {status['started_utc']}") - sections[key] = sec - - cyp2d6 = cyp2d6_block(sample, sample_dir, sections) - # The repeat readings lean on the traits (R117H) and CPIC (UGT1A1) - # tables, whose sections come after the expansions one: read them again - # with those sections' states. - ex = sections["expansions"] - if ex["state"] in ("ok", "stale"): - ex["data"]["readings"] = str_readings({l["locus"]: l for l in ex["data"]["loci"]}, sample_dir, sample, - {k: sections[k]["state"] for k in ("traits", "cpic")}) - if sections["locityper"]["state"] in ("ok", "stale"): - primary = locityper_primary(sections) - for x in sections["locityper"]["data"]["loci"]: - x["primary"] = primary.get(x["locus"], "no other step calls this locus") - declared = (declared_sex or (status or {}).get("declared_sex") or manifest_value(manifest, "run", "declared_sex") - or sections["sample_qc"]["data"].get("declared_sex") or None) - cnvpytor_sex_filter(sections["cnvpytor"]["data"], declared) - - clinvar_date = manifest_value(manifest, "data", "clinvar_file_date") - summary = { - "schema_version": SCHEMA_VERSION, - "sample": sample, - "generated_utc": iso(datetime.now(tz=timezone.utc).timestamp()), - "run": { - "status_file": status is not None, - "started_utc": status["started_utc"] if status else None, - "declared_sex": declared, - }, - "databases": { - "clinvar_release": clinvar_date, - "cpsr_clinvar_release": manifest_value(manifest, "data", "cpsr_clinvar_release"), - "hla_database": manifest_value(manifest, "data", "hla_database"), - }, - "manifest": manifest or [], - "sections": sections, - "cyp2d6": cyp2d6, - # The read count is written by alignment, which a run from a BAM or - # a VCF never does, so it is no step left undone. - "not_run": [sections[k]["title"] + (f" (step {sections[k]['step']} failed)" if sections[k]["state"] == "failed" else "") - for k, *_ in SECTIONS if sections[k]["state"] in ("missing", "failed") and k not in NOT_A_STEP], - "not_assessed": NOT_ASSESSED + ([] if sections["prs"]["data"].get("adjusted") else [PRS_NOT_ADJUSTED]) - # Step 35b on a region subset (--paraphase_regions) may leave hba out - + ([] if {"HBA1", "HBA2"} <= {g["gene"] for g in sections["paralogs"]["data"].get("genes", []) - if g["region"] == "hba" and g["status"] == "called"} - or (sections["parascopy"]["data"].get("hba") or {}).get("status") == "called" - else [HBA_NOT_ASSESSED]) - + ([MEI_NOT_ASSESSED] if sections["mei"]["state"] == "missing" else []) - + ([] if sections["vntyper"]["state"] == "ok" - and sections["vntyper"]["data"].get("status") in ("positive", "possible", "negative") - else [VNTYPER_NOT_ASSESSED]) - + ([] if sections["mosaic"]["data"].get("status") == "assessed" else [MCA_NOT_ASSESSED]), - } - recheck = run_delta.read_recheck(os.path.join(sample_dir, "recheck", "state.json")) - if recheck: - summary["recheck"] = recheck - since = new_since(sample_dir, summary) - if since: - summary["new_since"] = since - return summary - - -def new_since(sample_dir, summary): - """{previous_utc, sections}: what changed since the newest run kept in history/ (generate-report.sh - keeps one per run), or None without one. A folder that cannot be read counts as none.""" - prev_dir = run_delta.newest_history(sample_dir) - if not prev_dir: - return None - try: - with open(os.path.join(prev_dir, "summary.json")) as f: - prev = json.load(f) - changes = run_delta.compare(prev, run_delta.read_manifest(os.path.join(prev_dir, "run_manifest.tsv")), - summary, run_delta.read_manifest(os.path.join(sample_dir, "run_manifest.tsv"))) - except (OSError, ValueError, AttributeError, TypeError, KeyError): - return None - return {"previous_utc": prev.get("generated_utc"), "sections": run_delta.changed_sections(changes)} - - -def main(argv=None): - argv = sys.argv[1:] if argv is None else argv - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--sample", required=True) - ap.add_argument("--sample-dir", required=True) - ap.add_argument("--declared-sex", help="the samplesheet's sex, when no run status or manifest records it") - ap.add_argument("--out", help="summary JSON (default: SAMPLE_DIR/summary.json)") - a = ap.parse_args(argv) - summary = collect(a.sample, a.sample_dir, a.declared_sex) - out = a.out or os.path.join(a.sample_dir, "summary.json") - tmp = out + ".tmp" - with open(tmp, "w") as f: - json.dump(summary, f, indent=1) - f.write("\n") - os.replace(tmp, out) - print(f"Summary written: {out}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/render_report.py b/bin/render_report.py index 84be25703..eac22b4f4 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -10,7 +10,8 @@ sections that name them (bin/results_index.py); the HTML report closes with the same list. -With --sample-dir it first collects the summary (bin/collect_summary.py) and +With --sample-dir it first collects the summary with og summary collect +(rust/crates/og-summary, through bin/collect_summary.py's collect()), which writes it to --json (default DIR/summary.json). Every number in both reports comes from that summary, so the two cannot disagree. A section whose result is from an earlier run (state "stale") is shown with that date and a STALE mark, @@ -3223,7 +3224,7 @@ def charts_json(s): # --- html page: chapters, contents, section frames ----------------------------- CSP = "default-src 'none'; img-src data:; style-src 'unsafe-inline'" -STEP_OF = {k: step for k, _, step, _ in collect_summary.SECTIONS} +STEP_OF = {k: step for k, _, step in collect_summary.SECTIONS} # A section's state from the states of its keys. Never green: a section that # ran is not a finding, and the ClinVar section must show no green at all. STATE_BADGE = {"ok": ("Complete", "state"), "stale": ("Stale", "yellow"), "failed": ("Failed", "red"), @@ -3484,8 +3485,11 @@ def main(argv=None): elif a.sample and a.sample_dir: if a.assets: collect_summary.ASSETS_DIR = a.assets - s = collect_summary.collect(a.sample, a.sample_dir, a.declared_sex) - write(a.json or os.path.join(a.sample_dir, "summary.json"), json.dumps(s, indent=1) + "\n") + try: + s = collect_summary.collect(a.sample, a.sample_dir, a.declared_sex, + out=a.json or os.path.join(a.sample_dir, "summary.json")) + except RuntimeError as e: + sys.exit(f"ERROR: {e}") else: ap.error("give --summary, or --sample and --sample-dir") # Only the HTML report and --search-index read the index: a text-only run diff --git a/modules/local/html_report/main.nf b/modules/local/html_report/main.nf index 845f470f6..2734c6ba1 100644 --- a/modules/local/html_report/main.nf +++ b/modules/local/html_report/main.nf @@ -2,14 +2,15 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ HTML_REPORT — The sample's HTML report, rendered by bin/render_report.py ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The same code scripts/24-html-report.sh runs: bin/collect_summary.py - reads the step outputs into summary.json (the bash step's name for it), + The same code scripts/24-html-report.sh runs: og summary collect + (rust/crates/og-summary, run by render_report.py through bin/og) reads + the step outputs into summary.json (the bash step's name for it), and render_report.py writes the report from it, so a number on the Nextflow report is the number the bash report shows for the same file. The inputs are the outputs of the steps that ran for this sample, as one list (main.nf joins them, so the report waits for each). They are linked - into the folder layout collect_summary.py reads (clinvar/, pharmcat/, + into the folder layout og summary collect reads (clinvar/, pharmcat/, cpic/, vep/, clinical/, slivar/, cpsr/, qc/, roh/, mito/, telomere/, hla_t1k/, hla_optitype/, kir/, pypgx/, annotsv/, traits/, chip/, couple/, ...); a step that did @@ -18,7 +19,7 @@ name is the user's. indexcov's folder becomes indexcov/ itself (the sex check reads indexcov/indexcov-indexcov.ped), and CPIC_LOOKUP's PharmCAT comparison goes beside the pypgx summary, where - collect_summary.py looks for it. assets/acmg_sf.tsv (the ACMG SF rules) + og summary collect looks for it. assets/acmg_sf.tsv (the ACMG SF rules) and assets/carrier_genes.tsv (the carrier list) and assets/callability_genes.tsv (the hard-to-read genes) come in the same list and go to assets/, which --assets names, as do assets/mito_confirmed.tsv @@ -69,7 +70,7 @@ process HTML_REPORT { def gvcf_suffix = gvcf_index.endsWith('.csi') ? 'csi' : 'tbi' def release = workflow.manifest.version ?: '' """ - # Where collect_summary.py looks for each output (bin/collect_summary.py) + # Where og summary collect looks for each output (rust/crates/og-summary) S=layout/${id} mkdir -p "\$S" place() { mkdir -p "\$S/\$1" && ln -s "\$(readlink -f "\$2")" "\$S/\$1/\$(basename "\$2")"; } @@ -153,7 +154,7 @@ process HTML_REPORT { --search-index ${id}_search_index.json \\ --charts-json ${id}_charts.json - printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml + printf '"%s":\\n python: %s\\n og: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" "\$(og --version)" > versions.yml """ stub: diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index d50bd251a..2a0cb9695 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -13,10 +13,16 @@ use std::fs; use std::path::{Path, PathBuf}; /// How deep the walk goes below the sample folder. The deepest file a -/// section reads today is three folders down; Nextflow's work folders are -/// not inside a sample folder. +/// section reads today is three folders down. pub const MAX_DEPTH: usize = 6; +/// The top-level folder the walk leaves out: `run-all.sh` launches +/// Nextflow in `/nextflow`, whose `work/` holds every task's +/// files and staged links (a whole VEP cache among them). No section +/// reads a file there, and the Python collector, which globbed each +/// section's own folder, never listed it. +pub const SKIP_TOP: &str = "nextflow"; + /// Every regular file of a sample folder, by its path relative to the /// folder (`/` between parts). Symlinks are followed, so the links the /// HTML_REPORT task stages count as the files they point to; the depth cap @@ -54,7 +60,7 @@ impl FolderIndex { }; if meta.is_file() { files.insert(rel); - } else if meta.is_dir() && depth + 1 < MAX_DEPTH { + } else if meta.is_dir() && depth + 1 < MAX_DEPTH && !(depth == 0 && rel == SKIP_TOP) { stack.push((path, rel, depth + 1)); } } @@ -210,6 +216,22 @@ mod tests { assert_eq!(ix.first(&pats(&["vcf/T.vcf.gz"])), None); } + #[test] + fn the_nextflow_launch_folder_is_left_out() { + let d = Dir::new("scan-nextflow"); + d.put("nextflow/work/ab/cd/vcf/S.vcf.gz", "x"); + d.put("nextflow/S_sample_qc.tsv", "x"); + d.put("qc/nextflow/S_sample_qc.tsv", "x"); + let ix = FolderIndex::walk(&d.0); + assert_eq!(ix.first(&pats(&["nextflow/work/ab/cd/vcf/S.vcf.gz"])), None); + assert_eq!(ix.first(&pats(&["nextflow/*"])), None); + // Only the top-level folder: a deeper one of that name is read. + assert_eq!( + ix.first(&pats(&["qc/nextflow/S_sample_qc.tsv"])), + Some("qc/nextflow/S_sample_qc.tsv") + ); + } + #[test] fn a_missing_folder_is_empty() { let ix = FolderIndex::walk(Path::new("/nonexistent/og-summary-scan")); diff --git a/scripts/24-html-report.sh b/scripts/24-html-report.sh index 234a5c92b..1d7f0c3f9 100755 --- a/scripts/24-html-report.sh +++ b/scripts/24-html-report.sh @@ -2,10 +2,10 @@ # 24-html-report.sh — Generate a self-contained HTML report of all pipeline results # Usage: ./scripts/24-html-report.sh # -# Reads every step's output once into ${SAMPLE}/summary.json -# (bin/collect_summary.py) and renders the HTML report from it -# (bin/render_report.py). generate-report.sh renders the text report from the -# same summary, so the two reports show the same values. A result from an +# Reads every step's output once into ${SAMPLE}/summary.json (og summary +# collect, run by bin/render_report.py) and renders the HTML report from it. +# generate-report.sh renders the text report from the same summary when +# nothing has changed since, so the two reports show the same values. A result from an # earlier run whose step was skipped or failed in the latest run-all.sh run is # marked stale with its date. No internet connection is needed to view it. # ${SAMPLE}_search_index.json lists every gene and rsID the results mention diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index b6e3081de..5e5d3161d 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -2,12 +2,14 @@ # generate-report.sh — Create a plain-text summary report of all pipeline results # Usage: ./scripts/generate-report.sh # -# Reads every step's output once into ${SAMPLE}/summary.json -# (bin/collect_summary.py) and renders the text report from it -# (bin/render_report.py); step 24 renders the HTML report from the same -# summary. A result from an earlier run whose step was skipped or failed in the -# latest run-all.sh run is marked stale with its date. Run after completing all -# (or some) pipeline steps. +# Renders the text report (bin/render_report.py) from ${SAMPLE}/summary.json, +# the summary og summary collect reads from every step's output. When step 24 +# wrote that summary and nothing it reads has changed since (in a run-all.sh +# run, step 24 ran just before), the text report is rendered from it as it is: +# one collect per run, and both reports carry the same values and time. +# Otherwise it collects the summary again first. A result from an earlier run +# whose step was skipped or failed in the latest run-all.sh run is marked +# stale with its date. Run after completing all (or some) pipeline steps. # # Then it writes what changed since the previous run of the sample # (og run delta, docs/delta-report.md): ${SAMPLE}_delta.txt, .html and @@ -34,14 +36,39 @@ if [ ! -f "${SAMPLE_DIR}/run_manifest.tsv" ]; then GENOME_DIR="$GENOME_DIR" bash "${PGP_ROOT}/bin/write_manifest.sh" "$SAMPLE" generate-report.sh fi +# summary_current: 0 when SAMPLE_DIR/summary.json is newer than everything a +# collect reads: every file of the sample folder except what is written from +# the summary or beside it (the reports, the search index, the charts, the +# delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch +# folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a +# summary that is not there, means collect again. +summary_current() { + local s="${SAMPLE_DIR}/summary.json" newer + [ -s "$s" ] && [ ! -L "$s" ] || return 1 + newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune -o ! -type d -newer summary.json \ + ! \( -path './logs/*' ! -path ./logs/run_status.tsv \) \ + ! -name "${SAMPLE}_report.html" ! -name "${SAMPLE}_report.txt" ! -name "${SAMPLE}_search_index.json" \ + ! -name "${SAMPLE}_charts.json" ! -name "${SAMPLE}_delta.txt" ! -name "${SAMPLE}_delta.html" \ + ! -name "${SAMPLE}_delta.json" -print -quit) || return 1 + [ -z "$newer" ] || return 1 + newer=$(find "${PGP_ROOT}/bin" "${PGP_ROOT}/bin/.og" "${PGP_ROOT}/assets" -maxdepth 1 ! -type d -newer "$s" -print -quit 2> /dev/null) \ + || return 1 + [ -z "$newer" ] +} + rm -f "$REPORT" -# assets/: the ACMG SF rules (acmg_sf.tsv) and the carrier list (carrier_genes.tsv). +if summary_current; then + echo "Rendering from summary.json: step 24 wrote it and nothing it reads has changed since" + from=(--summary "/genome/${SAMPLE}/summary.json") +else + # assets/: the ACMG SF rules (acmg_sf.tsv) and the carrier list (carrier_genes.tsv). + from=(--assets /pgp-assets --json "/genome/${SAMPLE}/summary.json") +fi run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" "${PYTHON_IMAGE}" \ python3 /pgp-bin/render_report.py \ - --assets /pgp-assets \ + "${from[@]}" \ --sample "$SAMPLE" \ --sample-dir "/genome/${SAMPLE}" \ - --json "/genome/${SAMPLE}/summary.json" \ -o "/genome/${SAMPLE}/${SAMPLE}_report.txt" if [ ! -f "$REPORT" ]; then @@ -55,7 +82,7 @@ echo "Report saved to: ${REPORT}" # What changed since the previous run, then keep this run as the next baseline. HISTORY="${SAMPLE_DIR}/history" # og picks the previous run (--newest-history): the newest history// -# that holds a summary.json, the rule bin/collect_summary.py's "New since" +# that holds a summary.json, the rule og summary collect's "New since" # follows too. A newer folder without one, from a copy that stopped half # way, is passed over. og is the static binary, so the python image runs it. echo "" From 88d42be7f8b1e8bf9c1e61af2c6de55179f27809 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:32:46 +0100 Subject: [PATCH 21/86] test(summary): check the summary through og summary collect The tests that called the Python section readers now read the section og summary collect writes for the same folder. Checks of a Python internal (the G2 reader, the probable CYP2D6 rule, the poly-C tracts, the imprinted regions, the repeat ranges table) move to og-summary's own cargo tests. The demo maker writes step 26b, 49 and 05 tables without the Python remnants. collect_summary.py imports re for half_call. Bead: pgp-bs9u.45 --- bin/ancestry_fine_report.py | 119 --------- bin/annotsv_postfilter.py | 117 --------- bin/chip_report.py | 62 ----- bin/clip_check.py | 39 --- bin/collect_summary.py | 1 + bin/genotype_rules_indels.py | 210 ---------------- bin/pgx_outside_calls.py | 55 ----- bin/traits_report.py | 70 ------ .../og-summary/src/collect/group_c/chrm.rs | 5 + .../og-summary/src/collect/group_c/roh.rs | 43 ++++ .../og-summary/src/collect/group_c/tests.rs | 21 ++ .../og-summary/src/collect/group_c/traits.rs | 155 ++++++++++++ rust/crates/og-summary/src/collect/pgx.rs | 58 +++++ rust/crates/og-summary/src/collect/scan.rs | 3 +- tests/demo/make_demo_sample.py | 143 +++++++---- tests/e2e/callability-2-genes.sh | 16 +- tests/e2e/fixitfelix-chr21-1.sh | 16 +- tests/fake-docker/hla-kir-toggle.sh | 17 +- tests/test_acmg_sf_rules.py | 38 +-- tests/test_archaic_report.py | 33 +-- tests/test_callability.py | 24 +- tests/test_chip_report.py | 89 +++++-- tests/test_clip_check.py | 56 ----- tests/test_collect_summary.py | 161 ++++++------- tests/test_copy_number.py | 18 +- tests/test_cross_engine_biallelic.py | 29 ++- tests/test_exomiser_report.py | 28 ++- tests/test_genotype_rules_indels.py | 228 ------------------ tests/test_hla_report.py | 12 +- tests/test_locityper_report.py | 9 +- tests/test_lpa_kiv2.py | 27 ++- tests/test_mei_report.py | 37 ++- tests/test_mito_extended.py | 5 +- tests/test_mocha_report.py | 35 ++- tests/test_paralog_loci.py | 19 +- tests/test_pypgx_genes.sh | 2 +- tests/test_render_report.py | 21 +- tests/test_report_charts.py | 36 ++- tests/test_run_state.py | 2 +- tests/test_str_ranges.py | 11 +- tests/test_traits_report.py | 52 +++- tests/test_traits_table.py | 19 +- tests/test_viral_report.py | 26 +- tests/test_vntyper_report.py | 3 +- tests/test_ystr_report.py | 22 +- 45 files changed, 887 insertions(+), 1305 deletions(-) delete mode 100644 bin/ancestry_fine_report.py delete mode 100755 bin/annotsv_postfilter.py delete mode 100755 bin/chip_report.py delete mode 100755 bin/clip_check.py delete mode 100755 bin/genotype_rules_indels.py delete mode 100755 bin/pgx_outside_calls.py delete mode 100755 bin/traits_report.py delete mode 100644 tests/test_clip_check.py delete mode 100644 tests/test_genotype_rules_indels.py diff --git a/bin/ancestry_fine_report.py b/bin/ancestry_fine_report.py deleted file mode 100644 index 12043750f..000000000 --- a/bin/ancestry_fine_report.py +++ /dev/null @@ -1,119 +0,0 @@ -"""ancestry_fine_report.py: what other Python still reads of step 26b. - -Step 26b's commands (panel-map, check-map, order, sample-bed, chroms and -report) run in the Rust binary as og ancestry-fine (rust/crates/og-ancestry). -This module keeps only the names two Python programs import: - - bin/collect_summary.py label, COLOURS and MIN_REFERENCES, for the - report's group names and painting legend - tests/demo/make_demo_sample.py label, CHROM_LEN, painting_shares, - write_rows and painting_png, for the docs - demo, which is built without og - -The group names, the colours and MIN_REFERENCES must stay the same as -rust/crates/og-ancestry/src/lib.rs; tests/test_ancestry_fine_report.py checks -them against what og writes. - -A group name is ours: a set of reference populations we put together and -named, not a nationality. docs/26b-ancestry-fine.md lists them. -""" -import os -import struct -import zlib - -LABELS = { - "continent": { - "AFR": "Africa", - "WEA": "West Eurasia (Europe, Middle East, North Africa)", - "CSA": "Central and South Asia", - "EAS": "East Asia", - "AMR": "Americas (Native American)", - "OCE": "Oceania", - }, - "region": { - "WAF": "West Africa", "EAF": "East Africa", "CAF": "Central African rainforest", "SAF": "Southern Africa", - "NWE": "Northwestern Europe", "NEE": "Northeastern Europe", "IBE": "Iberia", "ITA": "Italy", - "SAR": "Sardinia", "CAU": "Caucasus", "LEV": "Levant", "BDN": "Bedouin", "NAF": "North Africa", - "CSA": "Central and South Asia", "EAS": "East Asia", "SIB": "Siberia", - "AMR": "Americas (Native American)", "OCE": "Oceania", - }, -} - -# Painting colours of the continental groups, and grey where nothing is painted. -COLOURS = {"AFR": (230, 159, 0), "WEA": (0, 114, 178), "CSA": (0, 158, 115), "EAS": (204, 121, 167), - "AMR": (213, 94, 0), "OCE": (86, 180, 233)} -UNPAINTED = (225, 225, 225) - -# GRCh38 lengths of chr1 to chr22, for the painting picture. -CHROM_LEN = [248956422, 242193529, 198295559, 190214555, 181538259, 170805979, 159345973, 145138636, - 138394717, 133797422, 135086622, 133275309, 114364328, 107043718, 101991189, 90338345, - 83257441, 80373285, 58617616, 64444167, 46709983, 50818468] - -# The fewest unrelated reference people a group needs to be modelled -# (rust/crates/og-ancestry/src/lib.rs says why). -MIN_REFERENCES = 10 - - -def label(level, code): - return LABELS.get(level, {}).get(code, code) - - -def write_rows(path, header, rows): - tmp = path + ".tmp" - with open(tmp, "w") as f: - f.write("\t".join(header) + "\n") - for r in rows: - f.write("\t".join(str(x) for x in r) + "\n") - os.replace(tmp, path) - - -def chrom_key(c): - c = c[3:] if c.startswith("chr") else c - return (0, int(c)) if c.isdigit() else (1, c) - - -def painting_shares(rows): - total = sum(r[2] - r[1] for r in rows) - by = {} - for r in rows: - by[r[4]] = by.get(r[4], 0) + r[2] - r[1] - return total, {k: v / total for k, v in by.items()} if total else {} - - -def png_bytes(width, height, pixels): - """A plain RGB PNG of rows of (r, g, b) tuples.""" - raw = b"".join(b"\x00" + bytes(v for px in row for v in px) for row in pixels) - - def chunk(kind, data): - return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xffffffff) - - return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) - + chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b"")) - - -def painting_png(rows, width=1000, bar=6, gap=6): - """Chromosomes 1 to 22 as two bars each (the two haplotypes), drawn to - scale, each painted segment in its group's colour, the rest grey.""" - scale = (width - 20) / CHROM_LEN[0] - height = gap + 22 * (2 * bar + 1 + gap) - img = [[(255, 255, 255)] * width for _ in range(height)] - for i, n in enumerate(CHROM_LEN): - y0 = gap + i * (2 * bar + 1 + gap) - for hap in (0, 1): - y = y0 + hap * (bar + 1) - for yy in range(y, y + bar): - for x in range(10, 10 + int(n * scale)): - img[yy][x] = UNPAINTED - for chrom, start, end, hap, code in rows: - key = chrom_key(chrom) - if key[0] != 0 or not 1 <= key[1] <= 22: - continue - i = key[1] - 1 - y = gap + i * (2 * bar + 1 + gap) + (hap - 1) * (bar + 1) - x0 = 10 + int(start * scale) - x1 = max(x0 + 1, 10 + int(end * scale)) - colour = COLOURS.get(code, (0, 0, 0)) - for yy in range(y, y + bar): - for x in range(x0, min(x1, width - 10)): - img[yy][x] = colour - return png_bytes(width, height, img) diff --git a/bin/annotsv_postfilter.py b/bin/annotsv_postfilter.py deleted file mode 100755 index 7ef0bee4c..000000000 --- a/bin/annotsv_postfilter.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -# remnant: kept for collect_summary and the docs demo; deleted in lane R3 -"""annotsv_postfilter.py: what is left of step 05's class 4 and 5 checks once -they moved to `og sv postfilter` (rust/crates/og-steps/src/sv). Only the -names the reports and the docs demo still import are kept: the table's -column names, the FORMAT reader of its sample column, and the join of step -22's consensus support that the reports make when they are written. -`og dev remnants` (tests/parity/remnants.tsv) checks that nothing else -comes back. Standard library only. -""" -import gzip -import os - -CONSENSUS_BP = 1000 -KEEP = ["AnnotSV_ID", "ID", "SV_chrom", "SV_start", "SV_end", "SV_length", "SV_type", "Gene_name", "ACMG_class", - "AnnotSV_ranking_score"] -ADDED = ["Caller", "Exon_edge", "Exon_edge_detail", "Population_AF", "Population_source", "Common", "AnnotSV_class", - "Depth_support", "Rule", "Rule_detail"] - - -def open_text(path): - if path.endswith((".gz", ".bgz")): - return gzip.open(path, "rt", errors="replace") - return open(path, errors="replace") - - -def chrom_key(c): - """AnnotSV writes chromosomes without 'chr'; step 22 writes them with it.""" - return c[3:] if c.lower().startswith("chr") else c - - -def sv_type(row): - t = (row.get("SV_type") or "").upper() - return "DUP" if t.startswith("DUP") else t - - -def info_value(info, key): - for kv in (info or "").split(";"): - k, _, v = kv.partition("=") - if k == key: - return v - return "" - - -def sample_format(row): - """{FORMAT key: value} of the first sample column, the one after FORMAT - in AnnotSV's header; {} when the row has none.""" - cols = sample_columns(row) - if not cols: - return {} - return dict(zip((row.get("FORMAT") or "").split(":"), (row.get(cols[1]) or "").split(":"))) - - -def sample_columns(row): - """["FORMAT", ] of AnnotSV's header, the two columns - sample_format reads; [] when the row has no sample column.""" - keys = list(row) - if "FORMAT" not in keys: - return [] - i = keys.index("FORMAT") + 1 - return [] if i >= len(keys) or keys[i] == "Annotation_mode" else keys[i - 1:i + 1] - - -def read_consensus(path): - """(caller names, records) of step 22's consensus VCF; a record is - (chrom, pos, end, type, supp, supp_vec).""" - names, recs = [], [] - with open_text(path) as f: - for line in f: - if line.startswith("##"): - continue - c = line.rstrip("\n").split("\t") - if line.startswith("#CHROM"): - names = c[9:] - continue - if len(c) < 8: - continue - t = info_value(c[7], "SVTYPE").upper() - try: - pos = int(c[1]) - end = int(info_value(c[7], "END") or pos) - supp = int(info_value(c[7], "SUPP") or 0) - except ValueError: - continue - recs.append((chrom_key(c[0]), pos, end, "DUP" if t.startswith("DUP") else t, supp, - info_value(c[7], "SUPP_VEC"))) - return names, recs - - -def consensus_support(rows, path): - """Support and Support_callers for each checked row from the consensus - at path; None for both when there is no consensus file.""" - if not path or not os.path.isfile(path): - return [{"Support": None, "Support_callers": None} for _ in rows] - names, recs = read_consensus(path) - by = {} - for rec in recs: - by.setdefault((rec[0], rec[3]), []).append(rec) - out = [] - for r in rows: - best = None - try: - start, end = int(r["SV_start"]), int(r["SV_end"]) - except (KeyError, ValueError): - start = end = None - if start is not None: - for rec in by.get((chrom_key(r.get("SV_chrom", "")), sv_type(r)), []): - d = abs(rec[1] - start) + abs(rec[2] - end) - if abs(rec[1] - start) <= CONSENSUS_BP and abs(rec[2] - end) <= CONSENSUS_BP and (best is None or d < best[0]): - best = (d, rec) - if best: - vec = best[1][5] - callers = [names[i] if i < len(names) else f"caller {i + 1}" for i, x in enumerate(vec) if x == "1"] - out.append({"Support": best[1][4], "Support_callers": ",".join(callers)}) - else: - out.append({"Support": 1, "Support_callers": r.get("Caller") or "."}) - return out diff --git a/bin/chip_report.py b/bin/chip_report.py deleted file mode 100755 index 10bccb594..000000000 --- a/bin/chip_report.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""chip_report.py: what the reports still read of step 49 (clonal haematopoiesis) in Python. - -The step itself runs in Rust: `og chip prepare`, `og chip report` and -`og chip check-table` (rust/crates/og-chip, docs/49-chip.md). This module -keeps only the names bin/collect_summary.py and tests/demo/make_demo_sample.py -import: the two tables' columns, the gene table reader and the detection -limit, the smallest VAF a clone needs to be found with probability -DETECT_POWER, taken as at least MIN_ALT_READS reads with the change out of the -mean depth (binomial). og chip computes the same limit the same way. - -Standard library only (it runs in the plain python image). -""" -import math -import sys - -TABLE_COLUMNS = ["gene", "truncating", "missense", "note", "pmids"] -OUT_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", - "alt_reads", "depth", "filter", "status", "class", "note"] -DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] -MIN_ALT_READS = 3 # reads with the change a call needs, and the detection limit assumes -DETECT_POWER = 0.8 # the detection limit: the VAF found this often at the mean depth - - -def die(msg): - sys.stderr.write(f"ERROR: {msg}\n") - sys.exit(1) - - -def read_table(path): - """The rows of assets/chip_genes.tsv as dicts (og chip check-table checks the rest of its schema).""" - rows = [] - with open(path) as fh: - lines = [ln.rstrip("\n") for ln in fh if not ln.startswith("#") and ln.strip()] - if not lines or lines[0].split("\t") != TABLE_COLUMNS: - die(f"{path}: the header must be {' '.join(TABLE_COLUMNS)} (tab separated)") - for ln in lines[1:]: - f = ln.split("\t") - if len(f) != len(TABLE_COLUMNS): - die(f"{path}: '{ln[:60]}' has {len(f)} columns, not {len(TABLE_COLUMNS)}") - rows.append(dict(zip(TABLE_COLUMNS, f))) - return rows - - -def p_at_least(k, n, f): - """P(X >= k) for X ~ Binomial(n, f).""" - return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) - - -def detection_limit(depth, k=MIN_ALT_READS, power=DETECT_POWER): - """The smallest VAF found with probability `power` at `depth` (k reads needed); None below k reads.""" - n = int(depth) - if n < k: - return None - lo, hi = 0.0, 1.0 - for _ in range(60): - mid = (lo + hi) / 2 - if p_at_least(k, n, mid) >= power: - hi = mid - else: - lo = mid - return hi diff --git a/bin/clip_check.py b/bin/clip_check.py deleted file mode 100755 index 91dadaa2a..000000000 --- a/bin/clip_check.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -"""clip_check.py: the names bin/collect_summary.py reads of the cDNA read check. - -The cDNA read check itself is og cdna check (rust/crates/og-bam/src/cdna.rs, -docs/06-clinvar-screen.md): steps 06 and 17 run it, and it writes -_clinvar_cdna_check.tsv and _cpsr_cdna_check.tsv. What is -left here is what the report reads beside those tables: the table's columns, -the note of a flagged hit, CPSR's classes in one spelling and the variant of -a CPSR row. og's own copies of these are in cdna.rs; the parity cases of -og-bam keep the two the same. -""" -import re - -FLAG_NOTE = "possible cDNA/transgene contamination: check the reads" -COLUMNS = ["chrom", "pos", "ref", "alt", "alt_reads", "alt_clipped", "ref_reads", "ref_clipped", - "cdna_check", "cdna_note", "construct_site"] - - -def norm_class(v): - """A CPSR or ClinVar class in one spelling: 'Likely_Pathogenic' and 'LP' - -> 'likely pathogenic'; '' for an empty or NA cell.""" - v = (v or "").replace("_", " ").strip().lower() - return {"p": "pathogenic", "lp": "likely pathogenic", "b": "benign", "lb": "likely benign", - "na": "", ".": ""}.get(v, v) - - -PLP = ("pathogenic", "likely pathogenic") - - -def cpsr_variant(row): - """(chrom, pos, ref, alt) of a CPSR row from VAR_ID or GENOMIC_CHANGE; - None when neither can be read.""" - f = (row.get("VAR_ID") or "").split("_") - if len(f) >= 4 and f[-3].isdigit(): - return "_".join(f[:-3]), int(f[-3]), f[-2], f[-1] - m = re.match(r"^(\S+):g\.(\d+)([A-Za-z]+)>([A-Za-z]+)$", row.get("GENOMIC_CHANGE") or "") - if m: - return m.group(1), int(m.group(2)), m.group(3), m.group(4) - return None diff --git a/bin/collect_summary.py b/bin/collect_summary.py index a492746d8..55bffce51 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -18,6 +18,7 @@ import gzip import json import os +import re import subprocess import tempfile import zlib diff --git a/bin/genotype_rules_indels.py b/bin/genotype_rules_indels.py deleted file mode 100755 index b30aaf65b..000000000 --- a/bin/genotype_rules_indels.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env python3 -"""genotype_rules_indels.py: the APOL1 G2 deletion, read beside step 42's APOL1 rule. - -Step 42's rule table (assets/genotype_rules.tsv, bin/traits_report.py) reads -single-base sites only, so its APOL1 rule reads G1 (two missense changes) and -says nothing about G2 (rs71785313), a 6-base deletion of N388 and Y389 a few -bases away (Genovese et al. 2010, PMID 20647424). G1 and G2 never share a copy, -and two risk copies (G1/G1, G1/G2 or G2/G2) are the high-risk genotype. This -module reads G2 from the sample's VCF and puts the two together: - - read_g2(records, kind) the G2 copies (0, 1, 2) from the VCF records that - overlap G2, with a status: - called a G2 call, or reference blocks - (gVCF) or reference calls over - the whole G2 span: 0 copies - no_call_record a VCF without reference blocks and - no record over G2: no G2 call, but - a reference base cannot be told - from a position with no reads - not_covered a gVCF with no genotype over part - of the G2 span - not_read a no-call, a filtered G2 call, an - allele other than G2 inside its - span, a REF that is not the - GRCh38 base, or more than one - record calling G2 (one copy - written twice cannot be told - from two copies) - combine(rule_line, g2) the G1 + G2 result and its reading, from step 42's - APOL1 line (G0/G0, G0/G1, G1/G1) and read_g2's answer. - -A deletion in a repeat can be written at several positions. A record is G2 -when applying its allele to GRCh38_CONTEXT gives the same sequence as G2, so a -left-aligned and a right-shifted record both count. The pinned dbSNP record, -DBSNP, is what NCBI's Variation API gave for rs71785313 on 2026-11-06 (the VCF -fields of NC_000022.11:36265995:ATAATTATAA:ATAA); tests/e2e/genotype-rules-2-g2.sh -compares it with dbSNP's VCF of the build versions.env pins, and -tests/test_genotype_rules_indels.py checks the marker against it and against -GRCh38_CONTEXT. Standard library only. -""" -import re - -# rsID, chrom, VCF POS, REF, ALT of rs71785313 in dbSNP's GRCh38 VCF (left-aligned). -DBSNP = ("rs71785313", "chr22", 36265995, "AATAATT", "A") -G2 = {"rule": "APOL1", "name": "G2", "rsid": "rs71785313", "chrom": "chr22", "pos": 36265995, - "ref": "AATAATT", "alt": "A"} -# GRCh38 chr22:36265975-36266025, the plus strand (UCSC hg38, 2026-11-06). dbSNP's -# SPDI for G2 spans 36265996-36266005 (ATAATTATAA becomes ATAA): any 6-base -# deletion inside that span is the same sequence. -CONTEXT_START = 36265975 -GRCh38_CONTEXT = "AGAAGCTAAACATTCTCAACAATAATTATAAGATTCTGCAGGCGGACCAAG" -CONTEXT_END = CONTEXT_START + len(GRCh38_CONTEXT) - 1 -# The G2 span: the anchor base of the left-aligned record to the end of the -# ambiguous span. A genotype over every base of it reads G2. -SPAN = (36265995, 36266005) -G1_COPIES = {"G0/G0": 0, "G0/G1": 1, "G1/G1": 2} -SYMBOLIC = re.compile(r"^(<.*>|\*|\.)$") - - -def apply(pos, ref, alt): - """GRCh38_CONTEXT with ref at pos replaced by alt, or None when ref is not - the GRCh38 bases there or the record does not lie inside the context.""" - i = pos - CONTEXT_START - if i < 0 or pos + len(ref) - 1 > CONTEXT_END or GRCh38_CONTEXT[i:i + len(ref)] != ref: - return None - return GRCh38_CONTEXT[:i] + alt + GRCh38_CONTEXT[i + len(ref):] - - -def marker_errors(m=G2, dbsnp=DBSNP): - """The problems of the G2 marker, as messages: against the pinned dbSNP - record (a swapped pair named as such) and against GRCh38_CONTEXT.""" - errs = [] - rec = (m["rsid"], m["chrom"], m["pos"], m["ref"], m["alt"]) - if rec != tuple(dbsnp): - if (m["ref"], m["alt"]) == (dbsnp[4], dbsnp[3]): - errs.append(f"{m['rsid']}: ref and alt are swapped (marker {m['ref']}>{m['alt']}, " - f"dbSNP REF {dbsnp[3]})") - else: - errs.append(f"{m['rsid']}: the marker is {m['chrom']}:{m['pos']} {m['ref']}>{m['alt']}, " - f"dbSNP {dbsnp[1]}:{dbsnp[2]} {dbsnp[3]}>{dbsnp[4]}") - hap = apply(m["pos"], m["ref"], m["alt"]) - if hap is None: - errs.append(f"{m['rsid']}: REF {m['ref']} is not the GRCh38 bases at {m['chrom']}:{m['pos']}") - elif len(GRCh38_CONTEXT) - len(hap) != 6: - errs.append(f"{m['rsid']}: the marker is not a 6-base deletion") - return errs - - -G2_HAPLOTYPE = apply(G2["pos"], G2["ref"], G2["alt"]) - - -def record_end(r): - """The last base a VCF record covers: INFO END for a reference block, else POS + len(REF) - 1.""" - m = re.search(r"(?:^|;)END=(\d+)", r[7]) if len(r) > 7 else None - return int(m.group(1)) if m else int(r[1]) + len(r[3]) - 1 - - -def overlaps(r): - """Whether a split VCF data line is on chr22 and covers part of the G2 span.""" - return (len(r) > 4 and r[0] == G2["chrom"] and r[1].isdigit() - and int(r[1]) <= SPAN[1] and record_end(r) >= SPAN[0]) - - -def read_g2(records, kind): - """G2 from the VCF records that overlap SPAN (other records are skipped). - kind is 'gvcf' (reference blocks: a base with no record has no reads) or - 'vcf' (variant calls only). Returns {status, copies, gq, text}; copies is - None unless status is called or no_call_record.""" - copies, covered, gqs, problems, seen, g2_records = 0, set(), [], [], False, 0 - for r in records: - if not overlaps(r): - continue - if len(r) < 10: - problems.append("a record over G2 has no sample column") - continue - seen = True - pos, ref = int(r[1]), r[3] - alleles = [ref] + r[4].split(",") - fmt = dict(zip(r[8].split(":"), r[9].split(":"))) - gt = re.split(r"[/|]", fmt.get("GT", ".")) - if any(not g.isdigit() for g in gt): - problems.append(f"a no-call at {r[0]}:{pos}") - continue - idx = [int(g) for g in gt] - if any(i >= len(alleles) for i in idx): - problems.append(f"a genotype at {r[0]}:{pos} names an allele the record lacks") - continue - if all(i == 0 for i in idx): - covered.update(range(pos, record_end(r) + 1)) - gqs.append(fmt.get("GQ", ".")) - continue - if r[6] not in ("PASS", "."): - problems.append(f"a call at {r[0]}:{pos} that failed the caller's filter ({r[6]})") - continue - n = 0 - for i in idx: - if i == 0: - continue - a = alleles[i] - inside = pos >= CONTEXT_START and record_end(r) <= CONTEXT_END - if inside and apply(pos, ref, ref) is None: - problems.append(f"REF {ref} at {r[0]}:{pos} is not the GRCh38 base (is the VCF on GRCh38?)") - break - hap = None if SYMBOLIC.match(a) or not inside else apply(pos, ref, a) - if hap == G2_HAPLOTYPE: - n += 1 - else: - problems.append(f"an allele other than G2 inside its span ({r[0]}:{pos} {ref}>{a})") - break - else: - if n: - g2_records += 1 - if g2_records > 1: - # Two records of the one deletion (say a left-aligned and a shifted copy of - # the same call) cannot be told from two G2 copies. - problems.append("more than one record calls the G2 deletion") - break - copies += n - covered.update(range(pos, record_end(r) + 1)) - gqs.append(fmt.get("GQ", ".")) - gq = ",".join(gqs) if gqs else "." - if problems: - return {"status": "not_read", "copies": None, "gq": gq, - "text": f"G2 (rs71785313) not read: {problems[0]}."} - if copies: - return {"status": "called", "copies": copies, "gq": gq, - "text": f"G2 (rs71785313): {copies} cop{'ies' if copies > 1 else 'y'} of the deletion called."} - if all(p in covered for p in range(SPAN[0], SPAN[1] + 1)): - return {"status": "called", "copies": 0, "gq": gq, - "text": "G2 (rs71785313): the reference over the whole deletion span, no G2 copy."} - if kind == "vcf" and not seen: - return {"status": "no_call_record", "copies": 0, "gq": ".", - "text": "G2 (rs71785313): no G2 deletion among the VCF's calls."} - return {"status": "not_covered", "copies": None, "gq": gq, - "text": "G2 (rs71785313) not read: no genotype over part of the deletion span."} - - -TWO_COPIES = ("Two APOL1 risk copies ({label}): the high-risk genotype of Genovese 2010, which went with " - "focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African " - "Americans. A risk factor, not a diagnosis.") - - -def combine(line, g2): - """{result, reading} of G1 and G2 together, from step 42's APOL1 line and read_g2's answer.""" - g1 = G1_COPIES.get(line.get("result")) if line.get("status") == "called" else None - if g1 is None and g2["status"] == "called" and g2["copies"] == 2: - # Two G2 copies fill both copies of the gene: G2/G2 whatever the G1 sites read. - return {"result": "G2/G2", "reading": TWO_COPIES.format(label="G2/G2") + " The G1 sites were not read, " - "but two G2 copies leave no copy for G1."} - if g1 is None: - return {"result": ".", "reading": "Not read: the G1 sites of the APOL1 rule were not read."} - if g2["copies"] is None: - return {"result": ".", "reading": "The G1 result above stands on its own."} - c = g2["copies"] - if g1 + c > 2: - return {"result": ".", "reading": f"{g1} G1 and {c} G2 copies do not fit two copies of the gene " - "(G1 and G2 never share a copy): not read. Check the calls."} - label = "/".join(["G0"] * (2 - g1 - c) + ["G1"] * g1 + ["G2"] * c) - if g1 + c == 2: - reading = TWO_COPIES.format(label=label) - elif g1 + c == 1: - reading = (f"One APOL1 risk copy ({label}): one copy did not raise the risk in Genovese 2010 " - "(a recessive pattern).") - else: - reading = "No G1 and no G2 copy: no APOL1 kidney risk copy." - if g2["status"] == "no_call_record": - label += " (no G2 call)" - reading += (" No G2 deletion was called, but this VCF has no reference blocks, so a G2 copy at a " - "position with no reads would not show.") - return {"result": label, "reading": reading} diff --git a/bin/pgx_outside_calls.py b/bin/pgx_outside_calls.py deleted file mode 100755 index 9b80128b8..000000000 --- a/bin/pgx_outside_calls.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# remnant: kept for collect_summary (and its test); deleted in lane R3 -"""pgx_outside_calls.py: what is left of step 36's outside calls once they -moved to `og pgx outside-calls`, `og pgx hla-concordance` and `og pgx -pypgx-summary` (rust/crates/og-steps/src/pgx_rules.rs and pypgx_summary.rs). -Only the rules the summary still imports are kept: what counts as a call, -the probable CYP2D6 call of pypgx alone, and the names of step 32's no-call -statuses. `og dev remnants` (tests/parity/remnants.tsv) checks that nothing -else comes back. Standard library only. -""" - -HLA_GENES = ("HLA-A", "HLA-B") -NO_CALL = {"", "none", "n/a", "na", "failed", "indeterminate", "not called", "unknown", "."} -PROBABLE = "probable (one caller: pypgx)" -# Step 32's summary: the status of a gene the reference cannot hold, and the -# quality flag of a call resting on a defining variant below the GQ floor. -NOT_CALLABLE = "not callable on this reference" -LOW_GQ = "low-confidence: check the reads" - - -def is_call(diplotype): - d = (diplotype or "").strip() - return d.lower() not in NO_CALL and "/" in d and ";" not in d - - -def same_diplotype(a, b): - """Equal up to the order of the two alleles.""" - def key(d): - return sorted(x.strip() for x in d.split("/")) - return key(a) == key(b) - - -def probable_cyp2d6(row, cyrius, depth_status): - """{'diplotype', 'phenotype', 'label'} for pypgx's CYP2D6 call when it is - the only usable call, else None. row: pypgx's CYP2D6 summary row (a - dict); cyrius: (genotype, filter), or None when Cyrius did not run; - depth_status: the depth check's status, or None without its file (step 32 - turns the row into Indeterminate when the check fails, so a call in the - row passed it). - - Not probable: no call, a usable Cyrius call (then the two callers decide), - a depth check that did not pass, a defining variant that is missing from - the VCF or not PASS (a summary written before these columns has none), a - star allele pypgx's table lacks, or a defining variant with a GQ below the - floor or no GQ.""" - if not row or not is_call(row.get("Diplotype")): - return None - if cyrius is not None and is_call(cyrius[0]) and cyrius[1] == "PASS": - return None - if depth_status is not None and depth_status != "ok": - return None - if (row.get("Defining_filter") or "") not in ("PASS", "none") or row.get("Quality") not in ("ok", "N/A"): - return None - return {"diplotype": row["Diplotype"].strip(), "phenotype": (row.get("Phenotype") or "").strip(), - "label": PROBABLE} diff --git a/bin/traits_report.py b/bin/traits_report.py deleted file mode 100755 index 0d8e67310..000000000 --- a/bin/traits_report.py +++ /dev/null @@ -1,70 +0,0 @@ -# remnant: kept for collect_summary, demo and catalog tests; deleted in lane R3. -"""Step 42 metadata still read by the Python report collectors. - -The executable implementation is og traits table, phase-sites and check. -Copy-aware marker readers are og cn marker-read. -""" -COLUMNS = ["chrom", "pos", "rsid", "gene", "ref", "alt", "trait", "hom_ref", "het", "hom_alt", - "ancestry", "note", "pmids"] -OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", - "note", "pmids"] -RULE_OUT_COLUMNS = ["rule", "gene", "trait", "rsids", "chrom", "pos", "genotypes", "gq", "phase", "result", - "status", "reading", "ancestry", "note", "pmids"] - -TRAITS_TABLE_RULES = ("APOE",) - -class TableError(Exception): - pass - - -def read_table(path): - """The rows of the traits table as dicts. Lines starting with # are comments; - the first other line is the header, which must be COLUMNS.""" - rows, header = [], None - with open(path, encoding="utf-8") as f: - for n, line in enumerate(f, 1): - line = line.rstrip("\n") - if not line or line.startswith("#"): - continue - cells = line.split("\t") - if header is None: - if cells != COLUMNS: - raise TableError(f"{path}:{n}: header is {cells}, expected {COLUMNS}") - header = cells - continue - if len(cells) != len(COLUMNS): - raise TableError(f"{path}:{n}: {len(cells)} columns, expected {len(COLUMNS)}") - row = dict(zip(COLUMNS, cells)) - row["line"] = n - rows.append(row) - if header is None: - raise TableError(f"{path}: no header line") - return rows - - -MARKERS = { - # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice - # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds - # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of - # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and - # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, - # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and - # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. - "LPA_KIV2_4925": { - "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", - "dbsnp": ("rs1484919544", "chr6", 160640667, "C", "T"), - "copies": (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932), - "context": "GACTCCTTACCTTGTTCGGAA", - "min_depth": 100, "min_alt": 5, "min_fraction": 0.008, "max_absent": 2, - }, - # SMN1 g.27134T>G is NM_000344.4:c.*3+80T>G in intron 7 (plus strand). - # SMN2 holds the same 21 bases 875,420 bases upstream, as at exon 7. - "SMN1_27134": { - "name": "SMN1 g.27134T>G", "chrom": "chr5", "ref": "T", "alt": "G", - "dbsnp": ("rs143838139", "chr5", 70952074, "T", "G"), - "copies": (70952074, 70076654), - "context": "TTTTTGAACATTTAAAAAGTT", - "min_depth": 10, "min_alt": 3, "min_fraction": 0.05, "max_absent": 1, - }, -} -MARKER_STATUSES = ("present", "not seen", "uncertain", "too little depth", "not assessed") diff --git a/rust/crates/og-summary/src/collect/group_c/chrm.rs b/rust/crates/og-summary/src/collect/group_c/chrm.rs index e232bccf6..68739a98a 100644 --- a/rust/crates/og-summary/src/collect/group_c/chrm.rs +++ b/rust/crates/og-summary/src/collect/group_c/chrm.rs @@ -467,6 +467,11 @@ mod tests { assert!(in_polyc(302, "A", "AC")); assert!(!in_polyc(300, "A", "G")); assert!(in_polyc(16189, "T", "C")); + // The tract's last base is in it, the next base is not; an SNV just + // before a tract is not, an insertion anchored there is. + assert!(in_polyc(315, "C", "T")); + assert!(!in_polyc(316, "C", "T")); + assert!(!in_polyc(302, "A", "G")); } #[test] diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index 489aa1a7e..f0fb24dc4 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -443,6 +443,49 @@ mod tests { use super::*; use crate::testutil::Dir; + /// assets/imprinted_regions.tsv holds the rows of [`IMPRINTED_REGIONS`] + /// (tests/test_collect_summary.py checked the Python copy the same way). + #[test] + fn imprinted_regions_are_the_asset_rows() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../assets/imprinted_regions.tsv" + ); + let text = std::fs::read_to_string(path).unwrap(); + let mut lines = text.lines().filter(|l| !l.starts_with('#')); + let head: Vec<&str> = lines.next().unwrap().split('\t').collect(); + let col = |name: &str| head.iter().position(|h| *h == name).unwrap(); + let num = |x: &str| (!x.is_empty()).then(|| x.parse::().unwrap()); + let rows: Vec<_> = lines + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + let span = num(f[col("start")]).zip(num(f[col("end")])); + ( + f[col("chrom")].to_owned(), + span, + f[col("region")].to_owned(), + f[col("maternal_upd")].to_owned(), + f[col("paternal_upd")].to_owned(), + f[col("pmid")].to_owned(), + ) + }) + .collect(); + let want: Vec<_> = IMPRINTED_REGIONS + .iter() + .map(|r| { + ( + r.0.to_owned(), + r.1, + r.2.to_owned(), + r.3.to_owned(), + r.4.to_owned(), + r.5.to_owned(), + ) + }) + .collect(); + assert_eq!(rows, want); + } + #[test] fn awk_reads_the_leading_number() { assert_eq!(awk_num("7000000"), 7_000_000.0); diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 4a93b5111..82afd2dc8 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -202,3 +202,24 @@ fn a_stale_section_says_from_which_run() { assert!(text(at(roh, &["note"])).contains("step 11 was skipped")); assert_eq!(at(roh, &["data", "largest_mb"]), &Value::Float(7.0)); } + +/// Every locus of the checkout's `bin/str_intermediate_ranges.tsv` is read +/// (tests/test_str_ranges.py checked the Python reader the same way). +#[test] +fn every_row_of_the_repeat_ranges_table_is_read() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../bin/str_intermediate_ranges.tsv" + ); + let rows = super::util::bin_rows(path).unwrap(); + let text = std::fs::read_to_string(path).unwrap(); + let loci: Vec<&str> = text + .lines() + .filter(|l| !l.starts_with('#') && !l.trim().is_empty()) + .skip(1) + .map(|l| l.split('\t').next().unwrap_or("")) + .collect(); + let read: Vec<&str> = rows.iter().map(|r| r["locus"].as_str()).collect(); + assert_eq!(read, loci); + assert!(loci.len() >= 3, "{loci:?}"); +} diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 6a553b36f..4e222e974 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -654,6 +654,161 @@ pub mod apol1 { assert!(g.text.contains("more than one record")); } + /// The GRCh38 bases `a..=b` of the pinned context. + fn ctx(a: i64, b: i64) -> &'static str { + &CONTEXT[(a - CONTEXT_START) as usize..=(b - CONTEXT_START) as usize] + } + + /// The tested record, with its filter and GQ. + fn call(pos: i64, r: &str, a: &str, gt: &str, filter: &str) -> Vec { + let mut x = rec(pos, r, a, gt, "."); + x[6] = filter.to_owned(); + x + } + + fn block(end: i64) -> Vec { + let mut x = rec(36_265_900, "A", "<*>", "0/0", &format!("END={end}")); + x[9] = "0/0:50".into(); + x + } + + // The checks of tests/test_genotype_rules_indels.py, which tested the + // Python G2 reader before this port replaced it. + #[test] + fn left_aligned_and_right_shifted_count_each_copy() { + let right = ( + 36_265_999, + ctx(36_265_999, 36_266_005), + ctx(36_265_999, 36_265_999), + ); + for (p, r, a) in [(POS, REF, ALT), right] { + for (gt, n) in [("0/1", 1), ("1|1", 2)] { + let g = read_g2(&[rec(p, r, a, gt, ".")], false); + assert_eq!((g.status, g.copies), ("called", Some(n)), "{p} {gt}: {g:?}"); + } + } + let two = read_g2( + &[ + rec(POS, REF, ALT, "0/1", "."), + rec(right.0, right.1, right.2, "0/1", "."), + ], + false, + ); + assert_eq!((two.status, two.copies), ("not_read", None)); + } + + #[test] + fn a_reference_block_keeps_its_gq_and_no_record_reads_zero() { + let g = read_g2(&[block(36_266_100)], true); + assert_eq!( + (g.status, g.copies, g.gq.as_str()), + ("called", Some(0), "50") + ); + let g = read_g2(&[], false); + assert_eq!((g.status, g.copies), ("no_call_record", Some(0))); + let g = read_g2(&[], true); + assert_eq!((g.status, g.copies), ("not_covered", None)); + let g = read_g2(&[block(36_266_000)], true); + assert_eq!( + g.status, "not_covered", + "a block that stops inside the span" + ); + } + + #[test] + fn anything_else_inside_the_span_is_not_read() { + let snv = ctx(36_266_001, 36_266_001); + let other = if snv == "G" { "C" } else { "G" }; + for (what, r) in [ + ("a no-call", call(POS, REF, ALT, "./.", "PASS")), + ("a filtered G2 call", call(POS, REF, ALT, "0/1", "RefCall")), + ( + "an SNV inside the span", + call(36_266_001, snv, other, "0/1", "PASS"), + ), + ( + "a 5-base deletion", + call(POS, ctx(POS, 36_266_000), "A", "0/1", "PASS"), + ), + ( + "a REF that is not GRCh38", + call(POS, "CATAATT", "C", "0/1", "PASS"), + ), + ] { + let g = read_g2(&[block(36_266_100), r], true); + assert_eq!((g.status, g.copies), ("not_read", None), "{what}: {g:?}"); + } + } + + #[test] + fn records_outside_the_span_are_skipped() { + let mut chr21 = rec(POS, REF, ALT, "1/1", "."); + chr21[0] = "chr21".into(); + let g = read_g2( + &[ + block(36_266_100), + call(36_266_100, "A", "G", "0/1", "PASS"), + chr21, + ], + true, + ); + assert_eq!((g.status, g.copies), ("called", Some(0))); + // Control: a 6-base deletion anchored right after the span is not G2. + let out = rec( + 36_266_006, + ctx(36_266_006, 36_266_012), + ctx(36_266_006, 36_266_006), + "1/1", + ".", + ); + assert_eq!(read_g2(&[block(36_266_100), out], true).copies, Some(0)); + } + + #[test] + fn combine_reads_every_count_and_refuses_three_copies() { + let called = |c: i64| G2 { + status: "called", + copies: Some(c), + gq: ".".into(), + text: "t".into(), + source: None, + source_first: false, + }; + for (g1, c, label) in [ + ("G0/G0", 0, "G0/G0"), + ("G0/G0", 1, "G0/G2"), + ("G0/G0", 2, "G2/G2"), + ("G0/G1", 0, "G0/G1"), + ("G0/G1", 1, "G1/G2"), + ("G1/G1", 0, "G1/G1"), + ] { + let (got, reading) = combine(g1, "called", &called(c)); + let word = match label.matches("G1").count() + label.matches("G2").count() { + 0 => "No G1 and no G2", + 1 => "One APOL1 risk copy", + _ => "Two APOL1 risk copies", + }; + assert_eq!(got, label); + assert!(reading.contains(word), "{label}: {reading}"); + } + let (got, reading) = combine("G1/G1", "called", &called(1)); + assert_eq!(got, "."); + assert!(reading.contains("never share"), "{reading}"); + assert_eq!(combine(".", "not_covered", &called(1)).0, "."); + let (got, reading) = combine(".", "not_covered", &called(2)); + assert_eq!(got, "G2/G2"); + assert!(reading.contains("Two APOL1 risk copies")); + let unread = G2 { + status: "not_read", + copies: None, + text: "G2 x.".into(), + ..called(0) + }; + let (got, reading) = combine("G0/G1", "called", &unread); + assert_eq!(got, "."); + assert!(reading.contains("stands on its own") && !reading.contains("G2 x.")); + } + #[test] fn combine_counts_both() { let g2 = read_g2(&[rec(POS, REF, ALT, "0/1", ".")], false); diff --git a/rust/crates/og-summary/src/collect/pgx.rs b/rust/crates/og-summary/src/collect/pgx.rs index d59db94c2..c271384a6 100644 --- a/rust/crates/og-summary/src/collect/pgx.rs +++ b/rust/crates/og-summary/src/collect/pgx.rs @@ -661,4 +661,62 @@ mod tests { ); assert!(json_load("[1]").is_ok()); } + + /// The checks of step 32's probable CYP2D6 call, from + /// tests/test_pgx_outside_calls.py, which tested the Python rule before + /// this port replaced it. + #[test] + fn pypgx_alone_is_probable_only_on_clean_evidence() { + let ev = |filter: &str, quality: &str| { + Some([ + "Normal Metabolizer".to_owned(), + filter.to_owned(), + quality.to_owned(), + ]) + }; + let cy = |g: &str, f: &str| Cyrius { + genotype: g.into(), + filter: Some(f.into()), + }; + let p = |e: Option<[String; 3]>, c: Option<&Cyrius>, depth: &str| { + probable("*1/*2", e.as_ref(), c, Some(depth)) + }; + assert_eq!( + p(ev("PASS", "ok"), None, "ok"), + Some(Probable { + diplotype: "*1/*2".into(), + phenotype: "Normal Metabolizer".into() + }) + ); + let none = cy("None", "Not_assigned_to_haplotypes"); + assert!( + p(ev("PASS", "ok"), Some(&none), "ok").is_some(), + "Cyrius ran, no call" + ); + assert!( + p(ev("PASS", "ok"), None, "unreliable").is_none(), + "a failed depth check" + ); + let call = cy("*1/*2", "PASS"); + assert!( + p(ev("PASS", "ok"), Some(&call), "ok").is_none(), + "two callers decide" + ); + for (filter, quality) in [ + ("PASS", "low-confidence: check the reads"), + ("not PASS: chr22:42128945 C>T RefCall", "ok"), + ("not in pypgx's table: *99", "ok"), + ("PASS", "GQ not in the VCF: check the reads"), + ] { + assert!( + p(ev(filter, quality), None, "ok").is_none(), + "{filter} {quality}" + ); + } + assert!( + p(None, None, "ok").is_none(), + "a summary without the columns" + ); + assert!(probable("None", ev("PASS", "ok").as_ref(), None, None).is_none()); + } } diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index 2a0cb9695..b81721f1f 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -60,7 +60,8 @@ impl FolderIndex { }; if meta.is_file() { files.insert(rel); - } else if meta.is_dir() && depth + 1 < MAX_DEPTH && !(depth == 0 && rel == SKIP_TOP) { + } else if meta.is_dir() && depth + 1 < MAX_DEPTH && !(depth == 0 && rel == SKIP_TOP) + { stack.push((path, rel, depth + 1)); } } diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index ff98d42fa..e55cb03d3 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -24,14 +24,14 @@ somalier, VerifyBamID2 and pgsc_calc files and the score folder `og prs format` writes (written here in its layout: og runs only inside the pipeline's Linux containers); - - step 33's mosaic rows come from og qc sex-loss, step 26b's painting - from bin/ancestry_fine_report.py, step 35b's table from + - step 33's mosaic rows come from og qc sex-loss, step 26b's table + and painting from og ancestry-fine report, step 35b's table from og paraphase table, step 39's from bin/couple_report.py and step 42's from bin/traits_report.py, each fed made-up inputs; step 46's table and step 35's HBA1/HBA2 table are what og cn call writes for made-up depths (HBA_TABLE and LPA_TABLE; tests/test_hba_copy_number.py and tests/test_lpa_kiv2.py check that og still writes them), step 49's depth - rows from bin/chip_report.py's detection_limit, with made-up counts; the + rows with og chip's detection limit, at made-up depths; the tables of steps 02, 40, 41, 45, 47, 48, 50, 51, 52 and 53 are written here, in the columns their writers use (og for 45 to 53), with the values those writers give for the made-up inputs named in each function. @@ -624,10 +624,11 @@ def write(path, recs, extra=()): f"{min(1, p * 9):.4e}\t{rng.uniform(0, 0.5):.4f}\t{rng.uniform(0, 0.1):.4f}\t{rng.randint(0, 9)}") put(os.path.join(d, "cnvpytor", f"{SAMPLE}_cnvs.txt"), "\n".join(lines) + "\n") - # Step 05's class 4 and 5 SVs, checked (bin/annotsv_postfilter.py's - # columns): none, as in most genomes. - import annotsv_postfilter - cols = annotsv_postfilter.KEEP + annotsv_postfilter.ADDED + # Step 05's class 4 and 5 SVs, checked (the columns og sv postfilter + # writes): none, as in most genomes. + cols = ["AnnotSV_ID", "ID", "SV_chrom", "SV_start", "SV_end", "SV_length", "SV_type", "Gene_name", "ACMG_class", + "AnnotSV_ranking_score", "Caller", "Exon_edge", "Exon_edge_detail", "Population_AF", + "Population_source", "Common", "AnnotSV_class", "Depth_support", "Rule", "Rule_detail"] rows = [] if has("annotsv"): # One class 4 deletion over an invented gene, from Manta, kept by step 05's rules. @@ -802,7 +803,10 @@ def callability(d, rng): written from the repository's catalog, so it always matches it. The ALT-only genes get no row, as in the real step.""" rows = ["#chrom\tstart\tend\tregion\t10X"] - for r in collect_summary.asset_rows("callability_genes.tsv"): + with open(os.path.join(REPO, "assets", "callability_genes.tsv"), encoding="utf-8") as f: + catalog = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", + quoting=csv.QUOTE_NONE)) + for r in catalog: if r["start"] == ".": continue start, end = int(r["start"]), int(r["end"]) @@ -852,41 +856,56 @@ def mosaic(d, rng): def ancestry_fine(d, rng): """Step 26b's table (ADMIXTURE fractions with their bootstrap range, then - the painted shares) and its painting, drawn by bin/ancestry_fine_report.py. - Mostly West Eurasian, as step 25's EUR placement above.""" - import ancestry_fine_report as afr + the painted shares) and its painting, written by og ancestry-fine report + from made-up ADMIXTURE and RFMix output. Mostly West Eurasian, as step + 25's EUR placement above.""" continent = {"WEA": 0.942, "CSA": 0.031, "AFR": 0.012, "EAS": 0.008, "AMR": 0.005, "OCE": 0.002} region = {"NWE": 0.463, "IBE": 0.274, "ITA": 0.118, "NEE": 0.052, "SAR": 0.009, "CAU": 0.011, "LEV": 0.008, "BDN": 0.002, "NAF": 0.005, "CSA": 0.031, "WAF": 0.007, "EAF": 0.003, "CAF": 0.001, "SAF": 0.001, "EAS": 0.006, "SIB": 0.002, "AMR": 0.005, "OCE": 0.002} - table = [] - for level, fr in (("continent", continent), ("region", region)): - for code, f in sorted(fr.items(), key=lambda kv: -kv[1]): - se = max(0.002, math.sqrt(f * (1 - f)) * rng.uniform(0.04, 0.08)) - table.append([level, code, afr.label(level, code), f"{f:.4f}", f"{se:.4f}", - f"{max(0.0, f - 1.96 * se):.4f}", f"{min(1.0, f + 1.96 * se):.4f}"]) - # The painting: each haplotype of chr1-22 in segments of 8 to 60 Mb, West - # Eurasian but for a few short ones. - rows = [] - for i, n in enumerate(afr.CHROM_LEN): - for hap in (1, 2): - pos = rng.randint(200_000, 1_500_000) - while pos < n - 2_000_000: - end = min(n - rng.randint(200_000, 1_500_000), pos + rng.randint(8_000_000, 60_000_000)) - code = "WEA" if rng.random() < 0.8 else rng.choice(["CSA", "CSA", "AFR"]) - if code != "WEA": - end = min(end, pos + rng.randint(3_000_000, 12_000_000)) - rows.append([f"chr{i + 1}", pos, end, hap, code]) + # GRCh38 lengths of chr1 to chr22. + chrom_len = [248956422, 242193529, 198295559, 190214555, 181538259, 170805979, 159345973, 145138636, + 138394717, 133797422, 135086622, 133275309, 114364328, 107043718, 101991189, 90338345, + 83257441, 80373285, 58617616, 64444167, 46709983, 50818468] + with tempfile.TemporaryDirectory() as work: + model, adm, paint, out = (os.path.join(work, x) for x in ("model", "admixture", "paint", "out")) + for x in (model, adm, paint): + os.makedirs(x) + levels = [] + for level, fr in (("continent", continent), ("region", region)): + codes = list(fr) + levels.append(f"{level}\t{len(codes)}\t{','.join(codes)}\n") + put(os.path.join(adm, f"{SAMPLE}_{level}.{len(codes)}.Q"), " ".join(f"{fr[c]:.6f}" for c in codes) + "\n") + # 20 bootstrap replicates, each fraction moved by a few percent of its spread. + boot = [] + for _ in range(20): + row = [max(0.0, fr[c] + rng.gauss(0, max(0.002, math.sqrt(fr[c] * (1 - fr[c])) * 0.06))) for c in codes] + boot.append(" ".join(f"{v / sum(row):.6f}" for v in row) + "\n") + put(os.path.join(adm, f"{SAMPLE}_{level}.boot.Q"), "".join(boot)) + put(os.path.join(model, "levels.tsv"), "".join(levels)) + # The painting: each chromosome in windows of 3 to 15 Mb, each haplotype + # West Eurasian but for a few windows. + groups = ["AFR", "CSA", "WEA"] + for i, n in enumerate(chrom_len): + c = f"chr{i + 1}" + rows = ["#Subpopulation order/codes: " + "\t".join(f"{g}={k}" for k, g in enumerate(groups)), + "#chm\tspos\tepos\tsgpos\tegpos\tn snps\t" + f"{SAMPLE}.0\t{SAMPLE}.1"] + pos, end_of = rng.randint(200_000, 1_500_000), n - rng.randint(200_000, 1_500_000) + while pos < end_of - 2_000_000: + end = min(end_of, pos + rng.randint(3_000_000, 15_000_000)) + haps = [2 if rng.random() < 0.85 else rng.choice([1, 1, 0]) for _ in (0, 1)] + rows.append(f"{c}\t{pos}\t{end}\t{pos / 1e6:.3f}\t{end / 1e6:.3f}\t100\t{haps[0]}\t{haps[1]}") pos = end - _, shares = afr.painting_shares(rows) - table += [["painting", code, afr.label("continent", code), f"{sh:.4f}", ".", ".", "."] - for code, sh in sorted(shares.items(), key=lambda kv: -kv[1])] - out = os.path.join(d, "ancestry_fine") - os.makedirs(out, exist_ok=True) - afr.write_rows(os.path.join(out, f"{SAMPLE}_ancestry_fine.tsv"), - ["level", "code", "group", "fraction", "se", "low", "high"], table) - with open(os.path.join(out, f"{SAMPLE}_painting.png"), "wb") as f: - f.write(afr.painting_png(rows)) + put(os.path.join(paint, f"{c}.msp.tsv"), "\n".join(rows) + "\n") + r = subprocess.run([og_binary(), "ancestry-fine", "report", "--sample", SAMPLE, "--model", model, + "--admixture", adm, "--paint", paint, "--panel", "demo", "--out", out], + stdout=subprocess.DEVNULL) + if r.returncode: + raise SystemExit(f"og ancestry-fine report failed (exit {r.returncode})") + dest = os.path.join(d, "ancestry_fine") + os.makedirs(dest, exist_ok=True) + for name in (f"{SAMPLE}_ancestry_fine.tsv", f"{SAMPLE}_painting.png"): + shutil.copy(os.path.join(out, name), os.path.join(dest, name)) def paralogs(d): @@ -1145,29 +1164,48 @@ def lpa(d, rng): put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), LPA_TABLE) +CHIP_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", + "alt_reads", "depth", "filter", "status", "class", "note"] +CHIP_DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] + + +def chip_limit(depth, k=3, power=0.8): + """og chip's detection limit: the smallest VAF that gives K or more reads + with the change with probability POWER at DEPTH (binomial, by bisection).""" + n = int(depth) + + def p_at_least(f): + return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) + lo, hi = 0.0, 1.0 + for _ in range(60): + mid = (lo + hi) / 2 + lo, hi = (lo, mid) if p_at_least(mid) >= power else (mid, hi) + return hi + + def chip(d, rng): """Step 49: no CHIP call, and one depth row per gene of assets/chip_genes.tsv - at a made-up mean depth near 31, with bin/chip_report.py's own detection limit.""" - import chip_report - table = chip_report.read_table(os.path.join(REPO, "assets", "chip_genes.tsv")) + at a made-up mean depth near 31, with og chip's detection limit.""" + with open(os.path.join(REPO, "assets", "chip_genes.tsv"), encoding="utf-8") as f: + lines = [x for x in f if not x.startswith("#") and x.strip()] rows = [] - for r in table: + for gene in (x.split("\t")[0] for x in lines[1:]): mean = rng.uniform(29.0, 33.0) - rows.append({"gene": r["gene"], "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), - "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_report.detection_limit(mean):.3f}", + rows.append({"gene": gene, "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), + "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_limit(mean):.3f}", "status": "covered"}) calls = [] if has("chip"): - # One small DNMT3A clone, above the detection limit at this depth (bin/chip_report.py's columns). + # One small DNMT3A clone, above the detection limit at this depth (og chip's columns). calls.append({"gene": "DNMT3A", "chrom": "chr2", "pos": "25234373", "ref": "C", "alt": "T", "consequence": "missense_variant", "protein_change": "p.R882H", "exon": "23", "vaf": "0.161", "alt_reads": "8", "depth": "50", "filter": "PASS", "status": "chip", "class": "hotspot", "note": "a published CHIP hotspot"}) - put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(chip_report.OUT_COLUMNS) + "\n" - + "".join("\t".join(c[k] for k in chip_report.OUT_COLUMNS) + "\n" for c in calls)) + put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(CHIP_COLUMNS) + "\n" + + "".join("\t".join(c[k] for k in CHIP_COLUMNS) + "\n" for c in calls)) put(os.path.join(d, "chip", f"{SAMPLE}_chip_depth.tsv"), - "\t".join(chip_report.DEPTH_COLUMNS) + "\n" - + "".join("\t".join(x[c] for c in chip_report.DEPTH_COLUMNS) + "\n" for x in rows)) + "\t".join(CHIP_DEPTH_COLUMNS) + "\n" + + "".join("\t".join(x[c] for c in CHIP_DEPTH_COLUMNS) + "\n" for x in rows)) def archaic(d, rng): @@ -1287,9 +1325,10 @@ def exomiser(d): def traits(d): - """Step 42: TRAIT_ALT_COPIES at the panel's positions, read by bin/traits_report.py.""" - import traits_report - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) + """Step 42: TRAIT_ALT_COPIES at the panel's positions, read by og traits table.""" + with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: + rows = list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", + quoting=csv.QUOTE_NONE)) unknown = sorted(r["rsid"] for r in rows if r["rsid"] not in TRAIT_ALT_COPIES | {"rs4410790": 0, "rs4481887": 0}) if unknown: raise SystemExit(f"assets/traits.tsv has rows TRAIT_ALT_COPIES does not give: {', '.join(unknown)}") diff --git a/tests/e2e/callability-2-genes.sh b/tests/e2e/callability-2-genes.sh index 960555b3e..6fdf87717 100755 --- a/tests/e2e/callability-2-genes.sh +++ b/tests/e2e/callability-2-genes.sh @@ -39,12 +39,18 @@ check_ge "the MAPQ 0 copy has reads of MAPQ 0 in the window" \ # reading SAMPLE_DIR_TABLE: "gene status" lines of a per-gene table read with the test catalog reading() { - python3 - "$CAT" "$1" "${REPO}/bin" <<'PY' -import csv, sys -sys.path.insert(0, sys.argv[3]) -import collect_summary + # og summary collect's callability section over the table, with the test + # catalog as the assets' callability_genes.tsv. + local t + t=$(mktemp -d "${CASE_TMP}/reading.XXXXXX") + mkdir -p "${t}/S/mosdepth" "${t}/assets" + cp "$CAT" "${t}/assets/callability_genes.tsv" + ln -s "$1" "${t}/S/mosdepth/S_callability.bed.gz" + "${REPO}/bin/og" summary collect --sample S --sample-dir "${t}/S" --assets "${t}/assets" --out "${t}/summary.json" > /dev/null + python3 - "$CAT" "${t}/summary.json" <<'PY' +import csv, json, sys rows = list(csv.DictReader(open(sys.argv[1]), delimiter="\t")) -d = collect_summary.callability(rows, sys.argv[2]) +d = json.load(open(sys.argv[2]))["sections"]["callability"]["data"] bad = {g["gene"]: g for g in d["not_read"]} for r in rows: g = bad.get(r["gene"]) diff --git a/tests/e2e/fixitfelix-chr21-1.sh b/tests/e2e/fixitfelix-chr21-1.sh index dc60e7dda..15751896f 100644 --- a/tests/e2e/fixitfelix-chr21-1.sh +++ b/tests/e2e/fixitfelix-chr21-1.sh @@ -181,12 +181,18 @@ grep -v '^#' "${REPO}/assets/callability_genes.tsv" \ | awk -F'\t' 'NR == 1 || $1 == "KCNE1" || $1 == "CBS" || $1 == "U2AF1" || $1 == "CRYAA"' > "$CAT" check_eq "the four genes' rows of assets/callability_genes.tsv" "$(tail -n +2 "$CAT" | cut -f1 | sort | tr '\n' ' ')" "CBS CRYAA KCNE1 U2AF1 " reading() { - python3 - "$CAT" "$1" "${REPO}/bin" <<'PY' -import csv, sys -sys.path.insert(0, sys.argv[3]) -import collect_summary + # og summary collect's callability section over the table, with the test + # catalog as the assets' callability_genes.tsv. + local t + t=$(mktemp -d "${CASE_TMP}/reading.XXXXXX") + mkdir -p "${t}/S/mosdepth" "${t}/assets" + cp "$CAT" "${t}/assets/callability_genes.tsv" + ln -s "$1" "${t}/S/mosdepth/S_callability.bed.gz" + "${REPO}/bin/og" summary collect --sample S --sample-dir "${t}/S" --assets "${t}/assets" --out "${t}/summary.json" > /dev/null + python3 - "$CAT" "${t}/summary.json" <<'PY' +import csv, json, sys rows = list(csv.DictReader(open(sys.argv[1]), delimiter="\t")) -d = collect_summary.callability(rows, sys.argv[2]) +d = json.load(open(sys.argv[2]))["sections"]["callability"]["data"] bad = {g["gene"] for g in d["not_read"]} print(" ".join(f"{r['gene']}={'not_read' if r['gene'] in bad else 'read'}" for r in rows)) PY diff --git a/tests/fake-docker/hla-kir-toggle.sh b/tests/fake-docker/hla-kir-toggle.sh index 3ef604d32..8da798813 100755 --- a/tests/fake-docker/hla-kir-toggle.sh +++ b/tests/fake-docker/hla-kir-toggle.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Step 08 run with KIR=true, then again without it, as after a realignment: # the second run must not leave the first run's KIR genotype files, or the -# report (collect_summary.kir_genotypes reads kir_t1k/_kir_genotype.tsv -# or kir/_kir_genotype.tsv, where the Nextflow pipeline publishes it, -# whenever it exists) shows them as this run's. A run that stops early, with +# report (og summary collect reads kir_t1k/_kir_genotype.tsv or +# kir/_kir_genotype.tsv, where the Nextflow pipeline publishes it, +# whenever it exists: rust/crates/og-summary, the hla section's kir) shows them +# as this run's. A run that stops early, with # the HLA database removed or on a missing BAM index, must not leave them either. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" @@ -52,10 +53,14 @@ mkdir -p "${G}/kir/IPD-KIR_${KIR_DB_RELEASE}" printf 'CC IPD-KIR Release Version %s\n' "$KIR_DB_RELEASE" > "${G}/kir/IPD-KIR_${KIR_DB_RELEASE}/kir.dat" K="${G}/sample1/kir_t1k" -# shown: what the report's summary reads for KIR (None = no KIR block). +# shown: True when a file the report's summary reads for KIR is there (the +# two paths of og summary collect's KIR reader, first one first). shown() { - python3 -c 'import sys; sys.path.insert(0, sys.argv[1]); import collect_summary as c; print(c.kir_genotypes(sys.argv[2], "sample1") is not None)' \ - "${REPO_ROOT}/bin" "${G}/sample1" + if [ -e "${G}/sample1/kir_t1k/sample1_kir_genotype.tsv" ] || [ -e "${G}/sample1/kir/sample1_kir_genotype.tsv" ]; then + echo True + else + echo False + fi } KIR=true run_expect 0 kir-on "${SCRIPTS}/08-hla-typing.sh" sample1 diff --git a/tests/test_acmg_sf_rules.py b/tests/test_acmg_sf_rules.py index 358e065a1..b3b07f6bd 100644 --- a/tests/test_acmg_sf_rules.py +++ b/tests/test_acmg_sf_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""The ACMG SF v3.3 reporting rules (assets/acmg_sf.tsv, bin/collect_summary.py -acmg_sf_tier) and the one-person carrier list (assets/carrier_genes.tsv), on +"""The ACMG SF v3.3 reporting rules (assets/acmg_sf.tsv, og summary collect's +ACMG SF tier, rust/crates/og-summary) and the one-person carrier list (assets/carrier_genes.tsv), on synthetic sample folders. Checks: @@ -33,6 +33,7 @@ class 5 deletion in MUTYH (no SV rule) is not; 9. without the assets folder the ClinVar hits and the clinical counts stay, and both lists say why they were not made. """ +import csv import gzip import os import shutil @@ -102,10 +103,23 @@ def sf_of(d): def sf_of_clinvar(d): - """The SF tier of a folder with ClinVar hits only (no clinical VCF): the - Nextflow layout gives sec_clinical the clinical VCF, so the tier is read - straight from the hits here.""" - return collect_summary.acmg_sf_tier(d, "S", None) + """The SF tier of a folder with ClinVar hits and a clinical VCF with no + record (the Nextflow layout: the clinical section is the VCF's count, and + the tier comes from the hits alone).""" + v = f"{d}/clinical/S_clinical.vcf.gz" + if not os.path.exists(v): + put(v, VCF_HEAD, gz=True) + return sf_of(d) + + +def catalog(folder): + """The rows of FOLDER/acmg_sf.tsv, and {gene: rule} with its rule cells + read (the variants as collect_summary.parse_alleles reads them).""" + with open(os.path.join(folder, "acmg_sf.tsv"), encoding="utf-8") as f: + rows = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) + return rows, {r["gene"]: {"inheritance": r["inheritance"], "report_if": r["report_if"], + "table_text": r["table_text"], "sv_cn": r["sv_cn"] == "yes", + "variants": collect_summary.parse_alleles(r["variants"])} for r in rows} def reported(sf, gene): @@ -122,8 +136,7 @@ def main(): # 1. the catalog p = os.path.join(REPO, "assets", "acmg_sf.tsv") try: - rows = collect_summary.asset_rows("acmg_sf.tsv") - rules = collect_summary.acmg_sf_rules() + rows, rules = catalog(os.path.join(REPO, "assets")) except Exception as e: # the pre-fix state: no catalog, no loader rows, rules = [], {} check("catalog: assets/acmg_sf.tsv loads", False, f"{type(e).__name__}: {e}") @@ -153,12 +166,7 @@ def main(): if os.path.isfile(p): with open(p) as f, open(os.path.join(dropped, "acmg_sf.tsv"), "w") as g: g.write("".join(x for x in f if not x.startswith("TTN\t"))) - keep = collect_summary.ASSETS_DIR - collect_summary.ASSETS_DIR = dropped - try: - n = len(collect_summary.acmg_sf_rules()) - finally: - collect_summary.ASSETS_DIR = keep + n = len(catalog(dropped)[1]) check("catalog control: a copy without TTN has 83 genes, not 84", n == 83, n) # 2. MUTYH: chr1 MUTYH p.Gly396Asp and p.Tyr179Cys (GRCh38) @@ -287,12 +295,10 @@ def main(): # 9. without the assets folder keep = collect_summary.ASSETS_DIR collect_summary.ASSETS_DIR = os.path.join(work, "no-assets") - collect_summary._ASSETS.clear() try: summ = collect_summary.collect("S", d) finally: collect_summary.ASSETS_DIR = keep - collect_summary._ASSETS.clear() cv, cl = summ["sections"]["clinvar"], summ["sections"]["clinical"] check("no assets: the ClinVar hits stay", cv["state"] == "ok" and cv["data"]["count"] == 4, cv["state"]) check("no assets: both lists say why they were not made", diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index cffe87e77..9fbeb2ea9 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -23,7 +23,7 @@ 3. report: Altai and Vindija segments merge into Neanderthal ones, a Denisova segment that overlaps one is not counted as Denisovan, segments under 50 kb or LOD 4 are dropped, the share is of the analysed span, and - the genes of the longest segments are named; bin/collect_summary.py + the genes of the longest segments are named; og summary collect reads the summary and stops on a damaged one, and bin/render_report.py prints both shares; 4. with IBDmix itself (IBDMIX_IMAGE, docker): a planted 600 kb stretch where @@ -387,29 +387,30 @@ def table_info(out, extra=()): import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 + +def archaic(d): + """The archaic section og summary collect writes for folder D.""" + return cs.collect("S1", d)["sections"]["archaic"] + + sdir = f"{TMP}/sample" os.makedirs(f"{sdir}/archaic") shutil.copy(f"{TMP}/out/S1_archaic.tsv", f"{sdir}/archaic/S1_archaic.tsv") -path, data = cs.sec_archaic(sdir, "S1") -check("collect_summary reads the summary", path and data.get("neanderthal_percent") == "5.00" - and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, data) -check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS)) +sec = archaic(sdir) +data = sec["data"] +check("og summary collect reads the summary", sec["state"] == "ok" and data.get("neanderthal_percent") == "5.00" + and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, sec) +check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS) + and sec["step"] == "43") lines = rr.archaic_lines(data) check("the report lines give both shares", lines[0].startswith("Neanderthal: 5.00% (0.5 Mb in 1 segments)") and lines[1].startswith("Denisovan: 2.00%"), lines) write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_percent\tlots\n") -try: - cs.sec_archaic(sdir, "S1") - check("a damaged summary is unreadable, not a share", False) -except cs.Unreadable: - check("a damaged summary is unreadable, not a share", True) +check("a damaged summary is unreadable, not a share", archaic(sdir)["state"] == "unreadable", archaic(sdir)) write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_segments\n") -try: - cs.sec_archaic(sdir, "S1") - check("a row cut after its key is unreadable, not a crash", False) -except cs.Unreadable: - check("a row cut after its key is unreadable, not a crash", True) -check("no file: the section is missing", cs.sec_archaic(f"{TMP}/nowhere", "S1") == (None, {})) +check("a row cut after its key is unreadable, not a crash", archaic(sdir)["state"] == "unreadable", archaic(sdir)) +nowhere = archaic(f"{TMP}/nowhere") +check("no file: the section is missing", (nowhere["state"], nowhere["data"]) == ("missing", {}), nowhere) # --- 4. the real IBDmix ------------------------------------------------------------- NEA = (2000000, 2600000) diff --git a/tests/test_callability.py b/tests/test_callability.py index c0696d9b7..2fd1aace9 100644 --- a/tests/test_callability.py +++ b/tests/test_callability.py @@ -6,7 +6,7 @@ tests/e2e/callability-1-catalog.sh rebuilds it and compares). Step 16b counts, per gene, the bases with 10 or more reads of MAPQ 20 or more (_callability.bed.gz, mosdepth's thresholds table), and -bin/collect_summary.py turns that into the report's section. +og summary collect (rust/crates/og-summary) turns that into the report's section. 1. the catalog: its header, one row per gene, known reasons, an HGNC id per row, a position on chr1-22, X or Y unless ALT-only, and owners as the build script's rule gives them (assets/paralog_loci.tsv, @@ -33,6 +33,7 @@ import csv import gzip import os +import shutil import subprocess import sys import tempfile @@ -117,9 +118,22 @@ def write_thresholds(path, catalog, zero=(), drop=()): # --- 2 to 4. the reader ---------------------------------------------------------------- +def og_callability(catalog, table): + """The callability data og summary collect reads from TABLE as step 16b's + table, in a sample folder of its own (og reads the repository's catalog, + which is CATALOG).""" + assert catalog is cat + sd = tempfile.mkdtemp(dir=os.path.dirname(table)) + os.makedirs(os.path.join(sd, "mosdepth")) + shutil.copy(table, os.path.join(sd, "mosdepth", "S_callability.bed.gz")) + sec = collect_summary.collect("S", sd)["sections"]["callability"] + return sec["data"] if sec["state"] == "ok" else {"not_read": [], "not_read_owned": [], "read": 0, + "not_on_reference": [], "state": sec} + + def alt_read_bug(catalog, table): """The planted bug: an ALT-only gene taken as read.""" - d = collect_summary.callability(catalog, table) + d = og_callability(catalog, table) d["not_read"] = [g for g in d["not_read"] if g["status"] != "not on this reference"] d["read"] += len(d["not_on_reference"]) return d @@ -160,12 +174,12 @@ def count_matches(reader, tsv): write_thresholds(tsv, cat, zero={MAPQ0, OWNED}) check("2. control: a reader that takes an ALT-only gene as read fails check 2", bool(mapq0_and_alt(alt_read_bug, tsv)), "the planted bug passed") - problems = mapq0_and_alt(collect_summary.callability, tsv) + problems = mapq0_and_alt(og_callability, tsv) check("2. a gene of MAPQ 0 reads is not read, a unique one is read, an ALT-only one is never read", not problems, problems) check("3. control: the planted bug's list differs from the catalog query", not count_matches(alt_read_bug, tsv)[0]) - ok, d = count_matches(collect_summary.callability, tsv) + ok, d = count_matches(og_callability, tsv) check("3. the genes not read with no owner are the catalog query's", ok, d["not_read"][:5]) check("3. owned genes not read are listed apart (HBA1 by the HBA count), GSTT1 is not one of them", any(g["gene"] == OWNED and g["owner"] == "cn_loci:HBA" for g in d["not_read_owned"]) @@ -209,7 +223,7 @@ def count_matches(reader, tsv): # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job # builds og in a temporary folder, so the tests run a copy that sits next # to the catalog, as bin/.og/og-* does in a checkout. - import atexit, shutil # noqa: E401 + import atexit # noqa: E401 _og_dir = tempfile.mkdtemp(prefix="og-cn-") atexit.register(shutil.rmtree, _og_dir, True) shutil.copy2(OG, os.path.join(_og_dir, "og")) diff --git a/tests/test_chip_report.py b/tests/test_chip_report.py index da506988c..42340069d 100644 --- a/tests/test_chip_report.py +++ b/tests/test_chip_report.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 """Step 49 (clonal haematopoiesis): og chip on synthetic bcftools output and on -assets/chip_genes.tsv, and what bin/chip_report.py still holds for the reports. +assets/chip_genes.tsv, and what the reports read of it. Without og (OG unset) only part A runs: - A. bin/chip_report.py's detection limit: about 14% at 30x (binomial, 3 - reads, 8 in 10), lower at 60x, none under 3x; the summary reader and - the report lines show the call and the limit (bin/collect_summary.py - sec_chip, bin/render_report.py chip_lines), the median limit included. + A. the detection limit, computed here as an independent binomial (3 + reads, 8 in 10): about 14% at 30x, lower at 60x, none under 3x; the + summary (og summary collect, which needs OG; without it part A stops + there) and the report lines show the call and the limit + (bin/render_report.py chip_lines), the median limit included. With OG naming the og binary, also: 1. og chip check-table passes assets/chip_genes.tsv; a malformed class token, a gene listed twice, a gene that counts no class and a bad @@ -23,7 +24,7 @@ 5. the depth table reads a base with no pileup line as depth 0, and a gene off the reference as not_on_reference; the summary sentence says the limit, that CHIP starts at 2%, that a negative result does not rule a - clone out, and that it is not a diagnosis; og's limit is the Python's; + clone out, and that it is not a diagnosis; og's limit is the binomial's; 6. prepare picks the MANE Select transcript of a synthetic Ensembl GFF3 (not the other one), renames the contigs (2 -> chr2), pads the coding exons by 10 bases, leaves out a contig the reference lacks, and stops on a gene @@ -32,6 +33,7 @@ Run: OG=rust/target/release/og python3 tests/test_chip_report.py """ +import math import os import subprocess import sys @@ -39,11 +41,14 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import chip_report as cr # noqa: E402 import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 TABLE = os.path.join(REPO, "assets", "chip_genes.tsv") +# The two tables og chip report writes, in column order. +OUT_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", + "alt_reads", "depth", "filter", "status", "class", "note"] +DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] OG = os.environ.get("OG", "") FAILS = 0 @@ -66,29 +71,65 @@ def og(*args): return subprocess.run([OG, "chip", *args], capture_output=True, text=True) +def p_at_least(k, n, f): + """P(X >= k) for X ~ Binomial(n, f).""" + return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) + + +def detection_limit(depth, k=3, power=0.8): + """The smallest VAF found with probability POWER at DEPTH (K reads needed), by + bisection; None below K reads. The test's own oracle for og chip's limit.""" + n = int(depth) + if n < k: + return None + lo, hi = 0.0, 1.0 + for _ in range(60): + mid = (lo + hi) / 2 + if p_at_least(k, n, mid) >= power: + hi = mid + else: + lo = mid + return hi + + +def table_genes(path): + """The genes of assets/chip_genes.tsv.""" + with open(path) as fh: + lines = [ln for ln in fh if not ln.startswith("#") and ln.strip()] + return [ln.split("\t")[0] for ln in lines[1:]] + + +def chip_section(d, sample): + """The chip section's data og summary collect reads from folder D.""" + if not OG: + print("skip: OG is not set (OG=rust/target/release/og): the summary needs og summary collect") + sys.exit(1 if FAILS or os.environ.get("OG_REQUIRED") else 0) + return cs.collect(sample, d)["sections"]["chip"]["data"] + + def chip_tables(d, sample, calls, depth): """A sample folder with the two tables, as og chip report writes them.""" os.makedirs(os.path.join(d, sample, "chip")) write(os.path.join(d, sample, "chip"), f"{sample}_chip.tsv", - "\t".join(cr.OUT_COLUMNS) + "\n" + "".join("\t".join(c) + "\n" for c in calls)) + "\t".join(OUT_COLUMNS) + "\n" + "".join("\t".join(c) + "\n" for c in calls)) write(os.path.join(d, sample, "chip"), f"{sample}_chip_depth.tsv", - "\t".join(cr.DEPTH_COLUMNS) + "\n" + "".join("\t".join(r) + "\n" for r in depth)) + "\t".join(DEPTH_COLUMNS) + "\n" + "".join("\t".join(r) + "\n" for r in depth)) return os.path.join(d, sample) -# A. what Python keeps -lim30, lim60 = cr.detection_limit(30), cr.detection_limit(60) +# A. the detection limit and what the reports read +lim30, lim60 = detection_limit(30), detection_limit(60) check("the 30x detection limit is between 13% and 16%", 0.13 < lim30 < 0.16, lim30) -check("P(>=3 reads) at that VAF is 0.8", abs(cr.p_at_least(3, 30, lim30) - 0.8) < 1e-6, cr.p_at_least(3, 30, lim30)) +check("P(>=3 reads) at that VAF is 0.8", abs(p_at_least(3, 30, lim30) - 0.8) < 1e-6, p_at_least(3, 30, lim30)) check("the 60x limit is lower", lim60 < lim30, (lim60, lim30)) -check("no limit below 3x", cr.detection_limit(2.9) is None) +check("no limit below 3x", detection_limit(2.9) is None) with tempfile.TemporaryDirectory() as d: call = ["DNMT3A", "chr2", "1150", "C", "T", "missense", "R882H", "2", "0.130", "6", "46", "PASS", "chip", "missense", "hotspot R882"] s = chip_tables(d, "S", [call], [["DNMT3A", "T2", "chr2", "392", "30.0", f"{lim30:.3f}", "covered"], ["JAK2", "T3", "chr9", "0", "", "", "not_on_reference"]]) - _, sec = cs.sec_chip(s, "S") - check("sec_chip reads the chip call and the median limit", + sec = chip_section(s, "S") + check("the summary reads the chip call and the median limit", [c["protein_change"] for c in sec.get("calls", [])] == ["R882H"] and abs(sec["median_limit"] - 0.137) < 0.001 and sec["not_covered"] == ["JAK2"], sec) text = "\n".join(rr.chip_lines(sec, "S")) @@ -96,13 +137,13 @@ def chip_tables(d, sample, calls, depth): "DNMT3A R882H: VAF 0.130 (6 of 46 reads)" in text and "about 14% or more" in text and "JAK2" in text, text) # With an even count of covered genes the median depth is the mean of the two # middle depths, and the reported limit must be the one at that depth. - e = chip_tables(d, "E", [], [[f"G{x}", "T", "chr1", "10", f"{x}.0", f"{cr.detection_limit(x):.4f}", "covered"] + e = chip_tables(d, "E", [], [[f"G{x}", "T", "chr1", "10", f"{x}.0", f"{detection_limit(x):.4f}", "covered"] for x in (20, 30, 60, 80)]) - _, sec4 = cs.sec_chip(e, "E") + sec4 = chip_section(e, "E") check("with 4 covered genes (20, 30, 60, 80x) the limit is the one at the median 45x", - sec4.get("median_depth") == 45 and abs(sec4["median_limit"] - cr.detection_limit(45)) < 1e-9, sec4) + sec4.get("median_depth") == 45 and abs(sec4["median_limit"] - detection_limit(45)) < 1e-9, sec4) check("no call reads 'No CHIP call.'", "No CHIP call." in rr.chip_lines({"calls": [], "median_limit": None}, "S")) - check("the table reader reads assets/chip_genes.tsv", "DNMT3A" in [r["gene"] for r in cr.read_table(TABLE)]) + check("assets/chip_genes.tsv holds DNMT3A", "DNMT3A" in table_genes(TABLE)) if not OG: print("skip: OG is not set (OG=rust/target/release/og): only part A ran") @@ -213,15 +254,15 @@ def status_of(c): p = og("report", "--genes", TABLE, "--gff", GFF_PATH, "--bed", bed, "--calls", EMPTY, "--depth", dep, "--sample", "S", "--out", out, "--depth-out", dout) lines = open(dout).read().splitlines() if p.returncode == 0 else [] - rows = {ln.split("\t")[0]: dict(zip(cr.DEPTH_COLUMNS, ln.split("\t"))) for ln in lines[1:]} + rows = {ln.split("\t")[0]: dict(zip(DEPTH_COLUMNS, ln.split("\t"))) for ln in lines[1:]} check("a base with no pileup line counts as depth 0 (mean 15)", rows.get("DNMT3A", {}).get("mean_depth") == "15.0", rows.get("DNMT3A")) check("a gene with no reads is too_shallow", rows.get("TET2", {}).get("status") == "too_shallow" and rows["TET2"]["mean_depth"] == "0.0", rows.get("TET2")) check("a gene off the reference is not_on_reference", rows.get("JAK2", {}).get("status") == "not_on_reference", rows.get("JAK2")) - check("every table gene has a depth row", sorted(rows) == sorted(r["gene"] for r in cr.read_table(TABLE))) - check("og's 15x limit is the Python's", rows.get("DNMT3A", {}).get("detection_limit_vaf") == f"{cr.detection_limit(15):.3f}", + check("every table gene has a depth row", sorted(rows) == sorted(table_genes(TABLE))) + check("og's 15x limit is the binomial's", rows.get("DNMT3A", {}).get("detection_limit_vaf") == f"{detection_limit(15):.3f}", rows.get("DNMT3A")) s = p.stdout.splitlines()[-1] if p.stdout else "" for want in ("about 26%", "15x", "2%", "does not rule", "Not a diagnosis"): @@ -274,11 +315,11 @@ def status_of(c): and "about 14%" in p3.stdout, p3.stdout + p3.stderr) lines = open(out).read().splitlines() if p3.returncode == 0 else [] check("the calls table has the columns in order and one chip row", - lines[:1] == ["\t".join(cr.OUT_COLUMNS)] and len(lines) == 2 and "\tchip\t" in lines[1], lines) + lines[:1] == ["\t".join(OUT_COLUMNS)] and len(lines) == 2 and "\tchip\t" in lines[1], lines) dl = open(dout).read().splitlines() if p3.returncode == 0 else [] check("the depth table has DNMT3A at 30x and JAK2 not on the reference", len(dl) == 3 and "\t30.0\t" in dl[1] and dl[2].endswith("not_on_reference"), dl) - _, sec = cs.sec_chip(os.path.join(d, "S"), "S") + sec = chip_section(os.path.join(d, "S"), "S") check("the reports read og's tables", [c["protein_change"] for c in sec.get("calls", [])] == ["R882H"] and sec["not_covered"] == ["JAK2"], sec) p4 = og("report", "--genes", genes, "--gff", out_gff, "--bed", out_bed, "--calls", os.path.join(d, "none.tsv"), diff --git a/tests/test_clip_check.py b/tests/test_clip_check.py deleted file mode 100644 index a6ba6b233..000000000 --- a/tests/test_clip_check.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -"""bin/clip_check.py: what is left of it after og cdna check took the read -check (rust/crates/og-bam, its unit tests and parity cases): the names -bin/collect_summary.py reads beside step 06's and step 17's tables. - - - norm_class spells every CPSR and ClinVar class one way; - - cpsr_variant reads VAR_ID, else GENOMIC_CHANGE, else nothing; - - COLUMNS and FLAG_NOTE are what og cdna check writes (the header and the - note of a flagged row of an og-bam parity golden). - -Run: python3 tests/test_clip_check.py -""" -import os -import sys - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import clip_check as cc # noqa: E402 - -FAILS = [] - - -def expect(desc, got, want): - if got == want: - print(f"PASS {desc}") - else: - print(f"FAIL {desc}: got {got!r}, want {want!r}") - FAILS.append(desc) - - -def main(): - expect("norm_class: underscores, case and spaces", cc.norm_class(" Likely_Pathogenic "), "likely pathogenic") - expect("norm_class: LP", cc.norm_class("LP"), "likely pathogenic") - expect("norm_class: NA and None are empty", (cc.norm_class("NA"), cc.norm_class(None)), ("", "")) - expect("norm_class: a class it does not know stays", cc.norm_class("VUS"), "vus") - expect("PLP", cc.PLP, ("pathogenic", "likely pathogenic")) - expect("cpsr_variant: VAR_ID", cc.cpsr_variant({"VAR_ID": "13_32340300_G_A"}), ("13", 32340300, "G", "A")) - expect("cpsr_variant: a contig with an underscore", cc.cpsr_variant({"VAR_ID": "HLA_A_5_C_T"}), - ("HLA_A", 5, "C", "T")) - expect("cpsr_variant: GENOMIC_CHANGE when VAR_ID does not read", - cc.cpsr_variant({"VAR_ID": ".", "GENOMIC_CHANGE": "2:g.47800000A>G"}), ("2", 47800000, "A", "G")) - expect("cpsr_variant: neither", cc.cpsr_variant({"GENOMIC_CHANGE": "2:g.A>G"}), None) - golden = os.path.join(REPO, "rust/crates/og-bam/tests/data/reads/want/check-spliced/files/out.tsv") - with open(golden) as fh: - lines = [line.rstrip("\n").split("\t") for line in fh] - expect("COLUMNS: the header og cdna check writes", lines[0], cc.COLUMNS) - expect("FLAG_NOTE: the note of its flagged rows", {r[9] for r in lines[1:] if r[8] == "flagged"}, {cc.FLAG_NOTE}) - if FAILS: - print(f"{len(FAILS)} check(s) failed") - return 1 - print("all checks passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index c5ffcc667..99be691f1 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -212,13 +212,33 @@ def og_postfilter(args): sys.path.insert(0, os.path.join(REPO, "bin")) sys.path.insert(0, os.path.join(REPO, "tests")) sys.path.insert(0, os.path.join(REPO, "tests", "schema")) -import clip_check # noqa: E402 import collect_summary # noqa: E402 import render_report # noqa: E402 import validate # noqa: E402 from og_binary import og_binary # noqa: E402 +def cdna_table(): + """The header og cdna check writes and the note of its flagged rows, from + an og-bam golden (rust/crates/og-bam/tests/data/reads).""" + with open(os.path.join(REPO, "rust/crates/og-bam/tests/data/reads/want/check-spliced/files/out.tsv")) as f: + rows = [line.rstrip("\n").split("\t") for line in f] + return rows[0], {r[9] for r in rows[1:] if r[8] == "flagged"}.pop() + + +CDNA_COLUMNS, CDNA_FLAG_NOTE = cdna_table() +# The columns of step 42's traits table (og traits panel). +TRAITS_OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", + "note", "pmids"] + + +def traits_rows(): + """The rows of assets/traits.tsv, step 42's panel, as dicts.""" + with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: + return list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", + quoting=csv.QUOTE_NONE)) + + def og_tool(*args): """og's exit code for a step tool that once lived in collect_summary.py (og qc sample, og prs table, og mito annotate): they write the tables @@ -256,6 +276,16 @@ def put(path, text, gz=False, age_days=0): os.utime(path, (t, t)) +# gzip's magic and header flags, then no deflate stream: not readable as gzip. +BROKEN_GZ = b"\x1f\x8b\x08\x00not gzip\n" + + +def put_bytes(path, data): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + + VCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" HITS = VCF_HEAD + ( "chr1\t10\t1\tC\tT\t50\tPASS\tGENEINFO=GENEA:11;CLNSIG=Pathogenic;CLNREVSTAT=criteria_provided,_single_submitter\tGT\t0/1\n" @@ -437,6 +467,15 @@ def render(d, s="S"): return summ, render_report.text_report(summ), render_report.html_report(summ) +def section(d, key, s="S", sex=None): + """One section of the summary og summary collect writes for folder D.""" + return collect_summary.collect(s, d, sex)["sections"][key] + + +# The five repeat loci the expansions section always lists, in this order. +EH_LOCI = ["HTT", "FMR1", "C9ORF72", "ATXN1", "DMPK"] + + def bgzf_block(data): """One BGZF block (a gzip member with the BC extra field) holding data.""" c = zlib.compressobj(6, zlib.DEFLATED, -15) @@ -449,7 +488,7 @@ def write_tabixed_vcf(path, header, blocks): """A bgzipped VCF of one BGZF block per item of blocks (lists of record lines, the header in the first block) and its .tbi: bin 0 of each sequence spans its lines (og mito annotate reads the bins), and the - linear index (bin/collect_summary.py reads that).""" + linear index (og summary collect reads that).""" raw, lin, span, coff = b"", {}, {}, 0 for i, lines in enumerate(blocks): data = (header if i == 0 else "").encode() @@ -505,7 +544,7 @@ def construct_scan_section(work, schema): put(f"{d}/clinvar/S_clinvar_hits.vcf", VCF_HEAD + ( f"chr7\t117548628\t1\tGTTT\tG\t50\tPASS\t{cftr}\tGT:GQ:DP\t1/.:45:30\n" f"chr21\t25897626\t2\tTC\tGA\t50\tPASS\t{app}\tGT:GQ:DP\t0/1:50:35\n")) - put(f"{d}/clinvar/S_clinvar_cdna_check.tsv", "\t".join(clip_check.COLUMNS) + "\n" + put(f"{d}/clinvar/S_clinvar_cdna_check.tsv", "\t".join(CDNA_COLUMNS) + "\n" "chr7\t117548628\tGTTT\tG\t\t\t\t\tnot_checked\tnot a SNV, MNV or simple indel\t\n" "chr21\t25897626\tTC\tGA\t12\t0\t10\t0\tclear\t\tAPP Swedish K670N/M671L\n") put(f"{d}/clinvar/S_clinvar_cdna_scan.tsv", "# cdna_scan: read 3 exon regions on 1 contigs; 1 loci listed\n" @@ -526,7 +565,6 @@ def construct_scan_section(work, schema): check("split poly-T: both reports print that wording, neither prints it as het", f"{half} Pathogenic" in txt and f"" in html and "chr7:117548628 GTTT>G het" not in txt, [l for l in txt.splitlines() if "117548628" in l]) - check("split poly-T: one copy of two (not hemizygous)", collect_summary.allele_copies("1/.") == (1, 2)) carriers = {x["gene"]: x for x in summ["sections"]["carrier"]["data"].get("carriers", [])} check("split poly-T: the carrier list words it the same way", carriers.get("CFTR", {}).get("genotype") == half, summ["sections"]["carrier"]["data"]) @@ -654,15 +692,12 @@ def clinvar_quality(work, schema): og_z = og_clinvar("hits", f"{z}/clinvar/S_clinvar_hits.vcf") check("call floor: a floor of GQ 0 and DP 0 marks nothing, so the mark above depends on the floor", cz["low_confidence"] == 0 and cz["count"] == 4 and "\tyes" not in (og_z or ""), (cz["low_confidence"], og_z)) - check("call floor: a hits file without the floor line uses GQ 30 and DP 10", - collect_summary.call_floor(f"{work}/cdna/S/clinvar/S_clinvar_hits.vcf") == {"GQ": 30, "DP": 10}) af = {g: (by[g]["af"], by[g]["common"]) for g in by} check("gnomAD AF: read through the tabix index from both BGZF blocks; above 1% is common; no VEP record is unknown", af == {"GENEA": (0.03, True), "GENEB": (0.0002, False), "GENEC": (0.0001, False), "GENED": (None, False)} and (by["GENEA"]["gnomad_exome_af"], by["GENEA"]["gnomad_genome_af"]) == (0.03, 0.02), af) - got_lin = collect_summary.tabix_linear_index(f"{d}/vep/S_vep.vcf.gz") - check("tabix: the linear index is read as written (chr2 starts in the second BGZF block)", - got_lin == lin and got_lin["chr2"][0] >> 16 > 0 and len(got_lin["chr1"]) == 3, (got_lin, lin)) + check("tabix: the test's index puts chr2 in the second BGZF block, so the AFs above came through it", + lin["chr2"][0] >> 16 > 0 and len(lin["chr1"]) == 3, lin) check("gnomAD AF: the summary names the VEP file", cv["frequency"]["source"] == "vep/S_vep.vcf.gz" and cv["frequency"]["note"] is None, cv["frequency"]) check("both reports: the AF and the common mark on the hit, unknown on the hit VEP lacks", @@ -741,9 +776,9 @@ def cpsr_beside_clinvar(work, schema): put(f"{d}/cpsr/S.cpsr.grch38.classification.tsv.gz", CPSR_COLS + CPSR_ROWS, gz=True) put(f"{d}/cpsr/S.cpsr.grch38.html", "\n") put(f"{d}/clinvar/S_clinvar_hits.vcf", FLOOR_HITS) - put(f"{d}/cpsr/S_cpsr_cdna_check.tsv", "\t".join(clip_check.COLUMNS) + "\n" + put(f"{d}/cpsr/S_cpsr_cdna_check.tsv", "\t".join(CDNA_COLUMNS) + "\n" "13\t32340300\tG\tA\t12\t0\t10\t0\tclear\t\n" - f"2\t47800000\tA\tG\t9\t8\t11\t0\tflagged\t{clip_check.FLAG_NOTE}\n") + f"2\t47800000\tA\tG\t9\t8\t11\t0\tflagged\t{CDNA_FLAG_NOTE}\n") put(f"{d}/run_manifest.tsv", "data\tclinvar_file_date\t20261001\ndata\tcpsr_clinvar_release\t20250309\n") summ, txt, html = render(d) c = summ["sections"]["cpsr"]["data"] @@ -861,11 +896,6 @@ def ranked(keep): rc == 0 and sorted(tags_nc) == [100, 200, 300, 400, 500, 700], sorted(tags_nc)) else: print("[NOTE] OG is not set, so og vcf csq-rank does not run here; its own tests pin these tags") - agree = collect_summary.csq_high_impact_genes(f"{d}/clinical/S_clinical.vcf.gz") - check("the summary agrees with og vcf csq-rank on which records are HIGH and which calls are weak", - {p_ for p_, t in want.items() if t[0] == "HIGH"} == {k[1] for k in agree} - and {p_ for p_, t in want.items() if t[3] == "weak"} == {k[1] for k, x in agree.items() if x["weak"]}, - (want, agree)) summ, txt, html = render(d) sf = summ["sections"]["clinical"]["data"]["acmg_sf"] by = {x["gene"]: x for x in sf["high_impact"]} @@ -1135,7 +1165,7 @@ def annotsv_section(work, schema): check("annotsv: without step 22 the summary validates", not validate.validate(schema, json.loads(json.dumps(summ))), validate.validate(schema, json.loads(json.dumps(summ)))) # a corrupt consensus: the rows stay, the note names the file - put(f"{d}/sv_merged/S_sv_consensus.vcf.gz", "not gzip") + put_bytes(f"{d}/sv_merged/S_sv_consensus.vcf.gz", BROKEN_GZ) sec = render(d)[0]["sections"]["annotsv"] check("annotsv: an unreadable consensus keeps the rows and says so", sec["state"] == "ok" and sec["data"]["count"] == 3 and "could not be read" in (sec["data"]["consensus_note"] or ""), @@ -1173,7 +1203,7 @@ def run(name, gt): "CLNREVSTAT=criteria_provided,_multiple_submitters,_no_conflicts\tGT:GQ:DP:PS\t1|0:50:30:45330000\n") with open(f"{d}/annotsv/S_sv_class4_5_checked.tsv") as f: head = f.readline().rstrip("\n").split("\t") - two = (collect_summary.sec_carrier(d, "S")[1].get("two_hits") or {}).get("genes", []) + two = (section(d, "carrier")["data"].get("two_hits") or {}).get("genes", []) return head, [x for x in two if x["gene"] == "MUTYH"] head, t = run("trans", "0|1") @@ -1186,14 +1216,10 @@ def run(name, gt): check("sv phase: a deletion genotyped 0/0 pairs with nothing", t == [], t) # A phased GT with one copy not called says nothing about cis or trans. - check("sv phase: '.|1' gives no copy, '0|1' gives copy 1", - (collect_summary.phase_haps(".|1"), collect_summary.phase_haps("0|1")) == (None, frozenset({1})), - (collect_summary.phase_haps(".|1"), collect_summary.phase_haps("0|1"))) - dl = {"inside": False, "chrom": "chr1", "start": 100, "end": 200, "gt": ".|1", "ps": "90"} - v = {"chrom": "chr1", "pos": 300, "gt": "0|1", "ps": "90", "copies": 1, "ploidy": 2} - verdict, why = collect_summary.pair_phase(dl, v) - check("sv phase: a deletion '.|1' and a hit '0|1' in one phase set are not called in cis", - verdict == "possibly biallelic" and "in cis" not in why, (verdict, why)) + _, t = run("halfcall", ".|1") + check("sv phase: a deletion '.|1' and a hit '1|0' in one phase set: phase unknown, not called in cis", + len(t) == 1 and t[0]["verdict"] == "possibly biallelic" and t[0]["why"].startswith("phase unknown") + and "in cis" not in t[0]["why"], t) def report_followups_sections(work, schema): @@ -1201,11 +1227,10 @@ def report_followups_sections(work, schema): flagged motif loci and the loci the repeat catalog leaves without thresholds reach both reports.""" import html as htmlmod - import traits_report # The genotype rules: TAS2R38 homozygous for the taster haplotype PAV # (G, G, C on the plus strand; rs1726866's G is the reference base). t = os.path.join(work, "rules", "S") - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) + rows = traits_rows() g = os.path.join(work, "rules_g.tsv") put(g, "chr7\t141972804\tT\tC\tC/C\t40\nchr7\t141972905\tG\t.\tG/G\t40\nchr7\t141973545\tC\tG\tG/G\t40\n" "chr19\t44908684\tT\t.\tT/T\t40\nchr19\t44908822\tC\t.\tC/C\t40\n") @@ -1315,7 +1340,7 @@ def made_up(name, tg, t, ugt, r117h=None, ugt_pc=None, stranger=False, traits_te {"LocusId": "CFTR", "NoThresholds": "the CFTR tract"}, {"LocusId": "UGT1A1", "NoThresholds": "the UGT1A1 repeat"}])) if r117h or traits_text: - put(f"{x}/traits/{name}_traits.tsv", traits_text or "\t".join(collect_summary.traits_report.OUT_COLUMNS) + put(f"{x}/traits/{name}_traits.tsv", traits_text or "\t".join(TRAITS_OUT_COLUMNS) + "\n" + f"rs78655421\tCFTR\tchr7\t117530975\tCFTR R117H\t{r117h}\tcalled\tr\tno\tn\t32404922\n", age_days=side_age) if ugt_pc: @@ -1763,7 +1788,7 @@ def parascopy_section(work, schema): and (pd.get("hba") or {}).get("alpha_globin_genes") == "4", pd.get("loci")) bound = os.path.join(g, "paralogs", "S_notch2_copy_number.tsv") put(bound, SMN_COLS + "chr1\t120000000\t120100000\tNOTCH2\tPASS\t<5\t30\t.\t.\t.\t*\n") - lb = collect_summary.parascopy_locus("notch2", bound, None) + lb = {x["table"]: x for x in section(g, "parascopy")["data"].get("loci", [])}.get("notch2", {}) check("loci: a bound such as <5 is not resolved, and the note names the whole-number rule", lb["status"] == "not resolved" and lb["cn"] is None and lb["text"].endswith(": no region with a whole copy number, filter PASS and quality 20 or more"), lb) @@ -1905,16 +1930,8 @@ def hla_optitype_kir_section(work, schema): def report_sections_mito_roh_telomere_cnv(work, schema): """ROH on autosomes outside the bands, the telomere read floor, CNVpytor's filtered count, the D-loop poly-C tracts and the chrM annotation.""" - import re as _re - # The masked bands: og's step 11 code holds the one copy that step 11 - # and the ROH module run (og roh summary); the summary's line agrees. - rs = open(os.path.join(REPO, "rust/crates/og-summary/src/collect/group_c/roh.rs")).read() - body = rs[rs.index("pub const MASKED_BANDS"):] - body = body[:body.index("];")] - rust_bands = " ".join(f"{c}:{int(a.replace('_', ''))}-{int(b.replace('_', ''))}" - for c, a, b in _re.findall(r'\("(chr\w+)", ([0-9_]+), ([0-9_]+)\)', body)) - check("bands: og's MASKED_BANDS is the line of bin/collect_summary.py", - rust_bands == collect_summary.MASKED_BANDS, rust_bands[:80]) + # The masked bands: og's step 11 code holds the one copy that step 11, + # the ROH module (og roh summary) and the summary run. for f in ("scripts/11-roh-analysis.sh", "modules/local/roh/main.nf"): text = open(os.path.join(REPO, f)).read() check(f"bands: {f} holds no copy of the bands and runs og roh summary", @@ -1943,15 +1960,6 @@ def report_sections_mito_roh_telomere_cnv(work, schema): "Left out in centromere and heterochromatin bands: 46.5 MB" in txt and "No population allele frequencies were used" in txt and "Segments on chrX/chrY, not counted: 1" in txt and "No population allele" in html, txt) - # Red control: masked_bands() with no bands, so nothing is masked. - orig = collect_summary.masked_bands - collect_summary.masked_bands = lambda text="": orig("") - try: - r0 = collect_summary.sec_roh(d, "S")[1] - finally: - collect_summary.masked_bands = orig - check("ROH red control: without the band mask the gvar segment is the largest again", - r0["largest_mb"] == 25.0 and len(r0["autosomal_over_5mb"]) == 4, r0) # Step 11's summary (og roh summary) on the same file lists the same segments. if OG: out = subprocess.run([OG, "roh", "summary", "--sample", "S", "--roh", f"{d}/vcf/S_roh.txt", @@ -1976,16 +1984,8 @@ def report_sections_mito_roh_telomere_cnv(work, schema): t["low_reads"] is low and t["depth"] == 30.0 and t["read_floor"] == 2000 and warned is low and ("too few telomeric reads" in html) is low and f"Intratelomeric reads: {n:,}" in txt, (t, txt)) put(f"{d}/telomere/S/S/S_summary.tsv", head + "S\ttumor\t620000000\t150\tn=7\t10\t40\t48-52\t150000000\t1.5\n") - saved = collect_summary.TELOMERE_FLOOR_PER_30X - collect_summary.TELOMERE_FLOOR_PER_30X = 0 - try: - t0 = collect_summary.sec_telomere(d, "S")[1] - finally: - collect_summary.TELOMERE_FLOOR_PER_30X = saved - check("telomere red control: with the floor at 0 the 40-read sample is not flagged, so the warning comes from it", - t0["low_reads"] is False, t0) put(f"{d}/telomere/S/S/S_summary.tsv", "PID\tsample\tintratelomeric_reads\ttel_content\nS\ttumor\t61873\t334\n") - t1 = collect_summary.sec_telomere(d, "S")[1] + t1 = section(d, "telomere")["data"] check("telomere: no total_reads or read length gives the count and says the check did not run", t1.get("intratelomeric_reads") == 61873 and "low_reads" not in t1 and "not checked" in t1["read_check"], t1) @@ -2040,18 +2040,6 @@ def report_sections_mito_roh_telomere_cnv(work, schema): in txt and "Length heteroplasmy in the D-loop poly-C tracts" in html, txt) check("mito: no annotation table says so, and why", "ClinVar and gnomAD: not annotated (no annotation table" in txt and "MITO_VARIANTS" in txt, txt) - saved = collect_summary.POLYC_TRACTS - collect_summary.POLYC_TRACTS = () - try: - m0 = collect_summary.sec_mito(d, "S")[1] - finally: - collect_summary.POLYC_TRACTS = saved - check("mito red control: without the tracts the count is 3 again", m0["heteroplasmic"] == 3, m0) - check("mito red control: without the tracts the poly-C calls are listed sites too, so the check above can fail", - len(m0["heteroplasmic_sites"]) == 3 and m0["heteroplasmic_sites"] != mi["heteroplasmic_sites"], m0) - check("mito: an SNV just before a tract is not in it, an insertion anchored there is", - not collect_summary.in_polyc(302, "A", "G") and collect_summary.in_polyc(302, "A", "AC") - and collect_summary.in_polyc(315, "C", "T") and not collect_summary.in_polyc(316, "C", "T")) # chrM annotation (step 20): ClinVar with chrM, tabixed; gnomAD's chrM sites. cv = "##fileformat=VCFv4.1\n##fileDate=2026-10-05\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n" @@ -2113,8 +2101,6 @@ def roh_upd_karyotype(work, schema): rows = list(csv.DictReader((l for l in f if not l.startswith("#")), delimiter="\t")) asset = [(r["chrom"], int(r["start"]) if r["start"] else None, int(r["end"]) if r["end"] else None, r["region"], r["maternal_upd"], r["paternal_upd"], r["pmid"]) for r in rows] - check("ROH catalog: assets/imprinted_regions.tsv holds the rows of collect_summary.IMPRINTED_REGIONS", - asset and asset == [tuple(x) for x in getattr(collect_summary, "IMPRINTED_REGIONS", [])], asset[:2]) check("ROH catalog: the six imprinted chromosomes, each row with a PMID", [r[0] for r in asset] == ["chr6", "chr7", "chr11", "chr14", "chr15", "chr20"] and all(x.isdigit() for r in asset for x in r[6].split(";")), asset) @@ -2145,7 +2131,7 @@ def roh_upd_karyotype(work, schema): validate.validate(schema, json.loads(json.dumps(summ)))) # Red control: the run on chr15 cut to 15q21 onwards misses 15q11-q13, so no row joins. put(f"{d}/vcf/S_roh.txt", upd.replace("chr15\t1\t101991189\t101991189", "chr15\t40000001\t101991189\t61991189")) - u0 = collect_summary.sec_roh(d, "S")[1].get("upd_flag") or {} + u0 = section(d, "roh")["data"].get("upd_flag") or {} check("ROH UPD red control: a chr15 run outside 15q11-q13 is flagged but joins no row", u0.get("chrom") == "chr15" and u0.get("imprinted") == [], u0) _, txt0, html0 = render(d) @@ -2164,7 +2150,7 @@ def roh_upd_karyotype(work, schema): for t in (txt5, html5)), txt5) # chr7 joins as a whole chromosome. put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr7\t70000001\t110000000\t40000000\t900\t50\n") - u7 = collect_summary.sec_roh(d, "S")[1].get("upd_flag") or {} + u7 = section(d, "roh")["data"].get("upd_flag") or {} check("ROH UPD: 40 Mb on chr7 joins the whole-chromosome Silver-Russell row", [x["maternal_upd"] for x in u7.get("imprinted", [])] == ["Silver-Russell syndrome"], u7) @@ -2185,12 +2171,12 @@ def roh_upd_karyotype(work, schema): check("ROH spread: with a band the F_ROH note says what the band is", all("The band is the row" in t for t in (txt, html)), txt) # Short runs only: F_ROH 0, no sign of related parents. put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t13000000\t3000000\t900\t50\n") - r = collect_summary.sec_roh(d, "S")[1] + r = section(d, "roh")["data"] check("ROH: runs under 5 Mb only give F_ROH 0 and no sign of related parents", r.get("f_roh") == 0 and r.get("relationship") == "no sign of related parents", r) # A run of exactly 5 Mb counts, as in step 11's summary (5 Mb or more). put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t15000000\t5000000\t900\t50\n") - r = collect_summary.sec_roh(d, "S")[1] + r = section(d, "roh")["data"] check("ROH: a run of exactly 5 Mb counts in F_ROH and the long-run list, as in step 11", r.get("long_mb") == 5.0 and len(r.get("autosomal_over_5mb") or []) == 1, r) @@ -2216,16 +2202,13 @@ def roh_upd_karyotype(work, schema): # somalier's ratios from 10 chrX sites or fewer name nothing: its own sex call needs more. for n in (10, "", "inf"): put(f"{k}/qc/S_sample_qc.tsv", qc.format(n=n, x="1.02", y="0.49")) - got = collect_summary.sec_sample_qc(k, "S")[1].get("karyotype") + got = section(k, "sample_qc")["data"].get("karyotype") check(f"karyotype: somalier's chrX 2.04, chrY 0.98 from x_sites '{n}' names nothing", got is None, got) # A run of infinite length makes the ROH file unreadable, not a crash in the band lookup. put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t13000000\tinf\t900\t50\n") - try: - r = collect_summary.sec_roh(d, "S") - except collect_summary.Unreadable as e: - r = e - check("ROH: an RG row of length inf is unreadable", isinstance(r, collect_summary.Unreadable), r) + r = section(d, "roh") + check("ROH: an RG row of length inf is unreadable", r["state"] == "unreadable", r) def main(): @@ -2390,9 +2373,10 @@ def main(): check("combined allele: the summary validates", not validate.validate(schema, json.loads(json.dumps(summ_cv))), validate.validate(schema, json.loads(json.dumps(summ_cv)))) - # 7. a corrupt file + # 7. a corrupt file: gzip's magic, then no gzip stream (a file named + # .gz that holds plain text is read as text: gzip goes by the magic) c = os.path.join(work, "corrupt", "S") - put(f"{c}/vcf/S.vcf.gz", "not gzip\n") + put_bytes(f"{c}/vcf/S.vcf.gz", BROKEN_GZ) put(f"{c}/clinvar/S_clinvar_hits.vcf", HITS) summ_c, txt_c, _ = render(c) check("corrupt VCF: section unreadable, the report still renders", @@ -2625,7 +2609,7 @@ def qc(sex, ped, x_n, x_het, x_hom_alt, declared, y_depth="0.0", mid="0.010"): ed.get("flagged") == [{"locus": "LOCUS_B", "repeat_count": "20/75", "status": "full_mutation", "sizing": "unknown"}], ed) check("Stranger: a record without STR_STATUS is counted apart, not flagged", ed.get("no_status") == 1, ed) - check("Stranger: the five key loci stay listed", [k["locus"] for k in ed["key_loci"]] == collect_summary.EH_LOCI, ed) + check("Stranger: the five key loci stay listed", [k["locus"] for k in ed["key_loci"]] == EH_LOCI, ed) check("Stranger: both reports list LOCUS_B full_mutation and the short-read caveat", "LOCUS_B 20/75 full_mutation" in txt_x and "" in html_x and "can be wrong at some loci" in txt_x and "can be wrong at some loci" in html_x, txt_x) @@ -2681,7 +2665,7 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): zd = summ_z["sections"]["expansions"]["data"] zl = {l["locus"]: l for l in zd.get("loci", [])} check("every locus: ten records give ten rows, the five key loci first", - [l["locus"] for l in zd.get("loci", [])][:5] == collect_summary.EH_LOCI and len(zl) == 10, zd.get("loci")) + [l["locus"] for l in zd.get("loci", [])][:5] == EH_LOCI and len(zl) == 10, zd.get("loci")) check("every locus: the catalog's normal maximum from STR_NORMAL_MAX", zl.get("HTT", {}).get("normal_max") == "26", zl.get("HTT")) check("sizing: a FLANKING-only call is flanking_only, not sized", zl.get("LOCUS_FL", {}).get("sizing") == "flanking_only", @@ -2731,7 +2715,7 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): put(f"{e}/expansion_hunter/S_eh.vcf", VCF_HEAD) txt_e = render(e)[1] check("text: a VCF without records still lists the five key loci as not in output", - all(f"{k:<8} not in output" in txt_e for k in collect_summary.EH_LOCI), txt_e) + all(f"{k:<8} not in output" in txt_e for k in EH_LOCI), txt_e) put(f"{z}/expansion_hunter/S_rfc1_motif.tsv", open(f"{z}/expansion_hunter/S_rfc1_motif.tsv").read(), age_days=3) zd2 = render(z)[0]["sections"]["expansions"]["data"] check("RFC1: a motif table older than Stranger's output is not read", @@ -2864,9 +2848,8 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): check("mosaic: an unknown mosaic_call is unreadable", sec_m["state"] == "unreadable" and "mosaic_call" in (sec_m["note"] or ""), sec_m) # 18. the traits panel (step 42) - import traits_report t = os.path.join(work, "traits", "S") - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) + rows = traits_rows() by_id = {r["rsid"]: r for r in rows} lct, ear = by_id["rs4988235"], by_id["rs17822931"] g = os.path.join(work, "traits_g.tsv") diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 5e1d250dc..04c53623c 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -905,7 +905,7 @@ def intervals(cat, read): check("step 35 interval block with a failed bedcov: exit 0, every row check_failed", out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines), (out.returncode, out.stderr[-400:])) - # collect_summary: step 35's section carries the interval rows with a change + # og summary collect: step 35's section carries the interval rows with a change import collect_summary as cs sd = os.path.join(t, "sample") os.makedirs(os.path.join(sd, "paralogs")) @@ -914,14 +914,14 @@ def intervals(cat, read): "homologous_regions\n") if os.path.exists(good): os.replace(good, os.path.join(sd, "paralogs", "S_cn_intervals.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] ci = data.get("cn_intervals") or {} - check("collect_summary: step 35's section lists the interval rows with a change (AZFc) and counts the rest", + check("summary: step 35's section lists the interval rows with a change (AZFc) and counts the rest", [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) os.remove(os.path.join(sd, "paralogs", "S_smn_copy_number.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] ci = (data or {}).get("cn_intervals") or {} - check("collect_summary: the interval rows stay when step 35 wrote no other table", + check("summary: the interval rows stay when step 35 wrote no other table", [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) @@ -1242,7 +1242,7 @@ def psv(cat, read): "to the couple screen (--cyp21a2).") in md} check(f"couple screen, {desc}: a gap line for each partner without a CYP21A2 table", got == want, (got, md[-600:])) - # collect_summary: step 35's section carries the CYP21A2 table + # og summary collect: step 35's section carries the CYP21A2 table import collect_summary as cs sd = os.path.join(t, "sample") os.makedirs(os.path.join(sd, "paralogs")) @@ -1251,15 +1251,15 @@ def psv(cat, read): "homologous_regions\n") if os.path.exists(tabs["deletion"]): os.replace(tabs["deletion"], os.path.join(sd, "paralogs", "S_cyp21a2_psv.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] c = data.get("cyp21a2") or {} - check("collect_summary: step 35's section carries the CYP21A2 table (deletion, 1 copy)", + check("summary: step 35's section carries the CYP21A2 table (deletion, 1 copy)", (c.get("finding"), c.get("cyp21a2_copies")) == ("deletion", "1"), c) # and the reports print why a table without MAPQ 0 reads is not assessed import render_report if os.path.exists(tabs["no_mapq0"]): os.replace(tabs["no_mapq0"], os.path.join(sd, "paralogs", "S_cyp21a2_psv.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] got = render_report.cyp21a2_line(data.get("cyp21a2")) check("report: the CYP21A2 line without MAPQ 0 reads is not assessed and says the BAM has few or no multi-mapping reads", ": not assessed (the BAM has few or no multi-mapping reads here" in got and "deletion," not in got, got) diff --git a/tests/test_cross_engine_biallelic.py b/tests/test_cross_engine_biallelic.py index 777232b48..2b89415be 100644 --- a/tests/test_cross_engine_biallelic.py +++ b/tests/test_cross_engine_biallelic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Two hits in one gene across callers (bin/collect_summary.py two_hits): a +"""Two hits in one gene across callers (og summary collect's two-hit rule): a called deletion plus a small variant, on synthetic sample folders. Nothing here describes a person: the calls are made up at real gene positions. @@ -111,7 +111,7 @@ def sample(work, name, hits, annotsv=None): def carrier(d): - return collect_summary.sec_carrier(d, "S")[1] + return collect_summary.collect("S", d)["sections"]["carrier"]["data"] def two(d, gene): @@ -119,7 +119,12 @@ def two(d, gene): def sf(d): - return collect_summary.acmg_sf_tier(d, "S", None) + """The SF tier of the folder: from the hits and the clinical VCF, which is + a VCF with no record when the case gives none (the Nextflow layout).""" + v = f"{d}/clinical/S_clinical.vcf.gz" + if not os.path.exists(v): + put(v, VCF_HEAD, gz=True) + return collect_summary.collect("S", d)["sections"]["clinical"]["data"].get("acmg_sf") or {} def main(): @@ -294,8 +299,8 @@ def main(): f"chr1\t{SNV[1]}\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|MUTYH|0.0001\tGT:GQ:DP:AD:PS\t" "0|1:50:30:15,15:45330000\n") d = sample(work, "high_trans", [], sv(DEL, "MUTYH", "1|0", ps=45330000)) - clin = put(f"{d}/clinical/S_clinical.vcf", csq) - rep = [x for x in collect_summary.acmg_sf_tier(d, "S", clin)["high_impact"] if x["gene"] == "MUTYH"] + put(f"{d}/clinical/S_clinical.vcf.gz", csq, gz=True) + rep = [x for x in sf(d)["high_impact"] if x["gene"] == "MUTYH"] check("a HIGH-impact hit phased in trans with the deletion: biallelic in the SF tier", len(rep) == 1 and (rep[0].get("two_hit") or "").startswith("biallelic with DEL"), rep) # one hit in cis, one in trans: only the trans one carries the note @@ -308,15 +313,17 @@ def main(): and "chr1:45330500 A>G" in rows and rows["chr1:45330500 A>G"] is None, rows) # an unreadable carrier list: the SF card says what was not read d = sample(work, "no_list", [hit(SNV, "MUTYH", "0/1")], sv(DEL, "MUTYH")) - real = collect_summary.carrier_genes - - def broken(): - raise collect_summary.Unreadable("carrier_genes.tsv row 1: broken") - collect_summary.carrier_genes = broken + broken = os.path.join(work, "assets-broken") + shutil.copytree(collect_summary.ASSETS_DIR, broken) + with open(os.path.join(broken, "carrier_genes.tsv")) as f: + text = f.read() + with open(os.path.join(broken, "carrier_genes.tsv"), "w") as f: + f.write(text.replace("gene\tchrom\tstart", "name\tchrom\tstart", 1)) + real, collect_summary.ASSETS_DIR = collect_summary.ASSETS_DIR, broken try: s = sf(d) finally: - collect_summary.carrier_genes = real + collect_summary.ASSETS_DIR = real check("an unreadable carrier list: a note on the SF card", "carrier_genes.tsv" in (s.get("two_hit_note") or "") and any("carrier_genes.tsv" in x for x in render_report.sf_apart_lines(s)), s.get("two_hit_note")) diff --git a/tests/test_exomiser_report.py b/tests/test_exomiser_report.py index b516b6a5d..c13e52e96 100644 --- a/tests/test_exomiser_report.py +++ b/tests/test_exomiser_report.py @@ -38,6 +38,21 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) +import collect_summary as cs # noqa: E402 + + +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + FAILS = [] @@ -318,7 +333,6 @@ def case_report(): def case_reports(): - import collect_summary as cs import render_report as rr with tempfile.TemporaryDirectory() as d: sd = os.path.join(d, "S1", "exomiser") @@ -326,14 +340,14 @@ def case_reports(): r = report(sd, ROWS) shutil.copy(os.path.join(sd, "top.tsv"), os.path.join(sd, "S1_exomiser_top.tsv")) os.rename(os.path.join(sd, "terms.tsv"), os.path.join(sd, "S1_exomiser_terms.tsv")) - p, data = cs.sec_exomiser(os.path.join(d, "S1"), "S1") + p, data = sec(os.path.join(d, "S1"), "S1", "exomiser") check("summary: the section reads the table", p is not None and r.returncode == 0) check("summary: ranks with their variants", [(g["rank"], g["gene"], len(g["variants"])) for g in data["genes"]] == [("1", "GENEA", 1), ("2", "GENEB", 2), ("3", "GENEC", 1)], str(data["genes"])) check("summary: the terms", data["terms"] == [{"id": "HP:0001156", "label": "Brachydactyly"}], str(data["terms"])) check("summary: the section is listed under step 48", - ("exomiser", "Symptom-driven ranking (Exomiser)", "48", cs.sec_exomiser) in cs.SECTIONS + ("exomiser", "Symptom-driven ranking (Exomiser)", "48") in cs.SECTIONS and "48" in cs.RUN_ALL_STEPS) text = "\n".join(rr.exomiser_text(data, "S1")) check("text report: rank, gene and variant", @@ -345,9 +359,9 @@ def case_reports(): bad = os.path.join(sd, "S1_exomiser_top.tsv") write(bad, "rank\tgene\n1\tGENEA\n") try: - cs.sec_exomiser(os.path.join(d, "S1"), "S1") + sec(os.path.join(d, "S1"), "S1", "exomiser") check("summary: a table without its columns is unreadable", False, "no error") - except cs.Unreadable: + except Unreadable: check("summary: a table without its columns is unreadable", True) # The -stub table of EXOMISER_REPORT reads as "no gene passed", not unreadable. import re @@ -359,9 +373,9 @@ def case_reports(): check("stub: the top table carries the report's columns", head == out_header, repr(head)) write(bad, head) try: - _, data = cs.sec_exomiser(os.path.join(d, "S1"), "S1") + _, data = sec(os.path.join(d, "S1"), "S1", "exomiser") check("stub: the summary reads it as no gene", data.get("genes") == [], str(data)) - except cs.Unreadable as e: + except Unreadable as e: check("stub: the summary reads it as no gene", False, str(e)) diff --git a/tests/test_genotype_rules_indels.py b/tests/test_genotype_rules_indels.py deleted file mode 100644 index 35fccfd72..000000000 --- a/tests/test_genotype_rules_indels.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -"""bin/genotype_rules_indels.py (APOL1 G2 beside step 42's APOL1 rule), and its path -through bin/collect_summary.py and bin/render_report.py. - -Checks: - 1. the G2 marker matches the pinned dbSNP record and the GRCh38 context; controls: - a marker with ref and alt swapped, one at another position and one whose REF is - not the GRCh38 bases each fail, naming the mistake; - 2. read_g2 on synthetic VCF records: the left-aligned dbSNP record and a - right-shifted record of the same deletion each count (het 1, hom 2); a gVCF - reference block over the span gives 0 copies, called, with its GQ; a - variant-only VCF with no record gives no_call_record; a gVCF with no record - gives not_covered; a no-call, a filtered G2 call, an SNV inside the span, a - 5-base deletion, a REF that is not GRCh38 and two het records of the one - deletion give not_read; a record outside - the span is skipped; control: a 6-base deletion just outside the span is not G2; - 3. combine: every G1 x G2 count gives its label and risk-copy reading; three risk - copies are not read; a G1 rule not called is not read, unless G2 has two copies - (G2/G2); G2 not read keeps the G1 result alone; a '.' result renders 'not read'; - no_call_record says the count rests on the absence of a call; - 4. collect_summary reads G2 from the sample's gVCF (bgzipped with a .tbi, and - plain gzip without one) or its VCF, and puts it on the APOL1 rule line only; - render_report shows "With G2" in the HTML and the text report; control: the - same folder with a reference gVCF gives another result. - -Run: python3 tests/test_genotype_rules_indels.py -""" -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import genotype_rules_indels as m # noqa: E402 -import render_report as rr # noqa: E402 - -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def ctx(a, b): - """GRCh38 bases a..b (1-based, inclusive) from the pinned context.""" - return m.GRCh38_CONTEXT[a - m.CONTEXT_START:b - m.CONTEXT_START + 1] - - -def rec(pos, ref, alt, gt, filt="PASS", info=".", gq="30"): - return ["chr22", str(pos), ".", ref, alt, "30", filt, info, "GT:GQ", f"{gt}:{gq}"] - - -LEFT = (36265995, "AATAATT", "A") # dbSNP's record -RIGHT = (36265999, ctx(36265999, 36266005), ctx(36265999, 36265999)) # the same deletion, shifted right -BLOCK = rec(36265900, "A", "<*>", "0/0", ".", "END=36266100", "50") - - -def marker_checks(): - check("1. the G2 marker matches dbSNP and GRCh38", m.marker_errors() == [], m.marker_errors()) - check("the G2 haplotype is GRCh38 with 6 bases deleted", len(m.GRCh38_CONTEXT) - len(m.G2_HAPLOTYPE) == 6) - swapped = dict(m.G2, ref="A", alt="AATAATT") - e = m.marker_errors(swapped) - check("control: ref and alt swapped fails, named as swapped", any("ref and alt are swapped" in x for x in e), e) - moved = dict(m.G2, pos=36265996) - e = m.marker_errors(moved) - check("control: another position fails against dbSNP and GRCh38", - any("dbSNP chr22:36265995" in x for x in e) and any("not the GRCh38 bases" in x for x in e), e) - e = m.marker_errors(dict(m.G2, ref="AATAATA"), dbsnp=("rs71785313", "chr22", 36265995, "AATAATA", "A")) - check("control: a REF that is not the GRCh38 bases fails even when dbSNP agrees", - any("not the GRCh38 bases" in x for x in e), e) - - -def read_checks(): - for name, (p, r, a) in (("left-aligned", LEFT), ("right-shifted", RIGHT)): - for gt, n in (("0/1", 1), ("1|1", 2)): - g = m.read_g2([rec(p, r, a, gt)], "vcf") - check(f"2. {name} G2 {gt}: {n} copies", (g["status"], g["copies"]) == ("called", n), g) - g = m.read_g2([rec(*LEFT, "0/1"), rec(*RIGHT, "0/1")], "vcf") - check("two het records of the one deletion (left-aligned and shifted): not read, never two copies", - (g["status"], g["copies"]) == ("not_read", None) and "more than one record" in g["text"], g) - g = m.read_g2([BLOCK], "gvcf") - check("a gVCF reference block over the span: 0 copies, called, its GQ", - (g["status"], g["copies"], g["gq"]) == ("called", 0, "50"), g) - g = m.read_g2([], "vcf") - check("a VCF with no record: no_call_record, 0 copies", (g["status"], g["copies"]) == ("no_call_record", 0), g) - g = m.read_g2([], "gvcf") - check("a gVCF with no record: not_covered", (g["status"], g["copies"]) == ("not_covered", None), g) - half = rec(36265900, "A", "<*>", "0/0", ".", "END=36266000", "50") - g = m.read_g2([half], "gvcf") - check("a gVCF block that stops inside the span: not_covered", g["status"] == "not_covered", g) - snv = ctx(36266001, 36266001) - other = "G" if snv != "G" else "C" - bad = {"a no-call": rec(*LEFT, "./."), "a filtered G2 call": rec(*LEFT, "0/1", "RefCall"), - "an SNV inside the span": rec(36266001, snv, other, "0/1"), - "a 5-base deletion": rec(36265995, ctx(36265995, 36266000), "A", "0/1"), - "a REF that is not GRCh38": rec(36265995, "CATAATT", "C", "0/1")} - for desc, r in bad.items(): - g = m.read_g2([BLOCK, r], "gvcf") - check(f"{desc}: not_read", (g["status"], g["copies"]) == ("not_read", None), g) - g = m.read_g2([BLOCK, rec(36266100, "A", "G", "0/1"), ["chr21", "36265995", ".", "AATAATT", "A", ".", - "PASS", ".", "GT", "1/1"]], "gvcf") - check("records outside the span or on another contig are skipped", (g["status"], g["copies"]) == ("called", 0), g) - # control: a 6-base deletion right after the span (anchor 36266006) is not G2 - out = rec(36266006, ctx(36266006, 36266012), ctx(36266006, 36266006), "1/1") - g = m.read_g2([BLOCK, out], "gvcf") - check("control: a 6-base deletion outside the span does not count as G2", g["copies"] == 0, g) - - -def combine_checks(): - want = {("G0/G0", 0): "G0/G0", ("G0/G0", 1): "G0/G2", ("G0/G0", 2): "G2/G2", ("G0/G1", 0): "G0/G1", - ("G0/G1", 1): "G1/G2", ("G1/G1", 0): "G1/G1"} - for (g1, c), label in want.items(): - res = m.combine({"status": "called", "result": g1}, {"status": "called", "copies": c, "text": "t"}) - risk = label.count("G1") + label.count("G2") - word = {0: "No G1 and no G2", 1: "One APOL1 risk copy", 2: "Two APOL1 risk copies"}[risk] - check(f"3. {g1} + {c} G2: {label}", res["result"] == label and word in res["reading"], res) - res = m.combine({"status": "called", "result": "G1/G1"}, {"status": "called", "copies": 1, "text": "t"}) - check("G1/G1 with a G2 copy (three risk copies): not read", res["result"] == "." and "never share" in res["reading"], - res) - res = m.combine({"status": "not_covered", "result": "."}, {"status": "called", "copies": 1, "text": "t"}) - check("the G1 rule not called: not read", res["result"] == ".", res) - res = m.combine({"status": "not_covered", "result": "."}, {"status": "called", "copies": 2, "text": "t"}) - check("the G1 rule not called with two G2 copies: G2/G2, two risk copies", - res["result"] == "G2/G2" and "Two APOL1 risk copies" in res["reading"], res) - res = m.combine({"status": "called", "result": "G0/G1"}, {"status": "not_read", "copies": None, "text": "G2 x."}) - check("G2 not read: the G1 result stands alone, without repeating G2's text", - res["result"] == "." and "stands on its own" in res["reading"] and "G2 x." not in res["reading"], res) - for res in (m.combine({"status": "called", "result": "G1/G1"}, {"status": "called", "copies": 1, "text": "t"}), - m.combine({"status": "not_covered", "result": "."}, {"status": "called", "copies": 1, "text": "t"})): - shown = rr.g2_result({"g2": dict(res, status="called", text="t")}) - check(f"render_report: a '.' result shows 'not read' ({res['reading'][:30]})", shown == "not read", shown) - res = m.combine({"status": "called", "result": "G0/G1"}, m.read_g2([], "vcf")) - check("no_call_record: the label says no G2 call and the reading why", - res["result"] == "G0/G1 (no G2 call)" and "no reference blocks" in res["reading"], res) - - -def bgzip_with_tbi(path, lines): - """A bgzipped VCF and its .tbi from bgzip and tabix when they exist, else None - (tests/e2e/genotype-rules-2-g2.sh covers the .tbi path with the pinned image).""" - if not (shutil.which("bgzip") and shutil.which("tabix")): - return None - with open(path[:-3], "w") as f: - f.writelines(lines) - subprocess.run(["bgzip", "-f", path[:-3]], check=True) - subprocess.run(["tabix", "-p", "vcf", path], check=True) - return path - - -def report_checks(): - import collect_summary as cs - import render_report as rr - from test_traits_report import traits_tables - head = "##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - g1 = [("chr22", "36265860", "A", "G", "A/G", "40"), ("chr22", "36265988", "T", "G", "T/G", "40")] - work = tempfile.mkdtemp() - try: - def folder(name, gvcf_lines, kind): - d = os.path.join(work, name) - os.makedirs(os.path.join(d, "traits")) - os.makedirs(os.path.join(d, "vcf")) - p = os.path.join(d, "traits", "S_genotype_rules.tsv") - g = os.path.join(work, name + "_g.tsv") - with open(g, "w") as f: - f.writelines("\t".join(x) + "\n" for x in g1) - traits_tables(g, os.path.join(d, "traits", "S_traits.tsv"), p) - v = os.path.join(d, "vcf", "S.g.vcf.gz" if kind == "gvcf" else "S.vcf.gz") - with gzip.open(v, "wt") as f: - f.write(head + "".join("\t".join(r) + "\n" for r in gvcf_lines)) - return d - - het = rec(*LEFT, "0/1") - d = folder("het", [het], "gvcf") - rows = cs.genotype_rules(d, "S") - apol1 = next(r for r in rows if r["rule"] == "APOL1") - others = [r["rule"] for r in rows if "g2" in r and r["rule"] != "APOL1"] - check("4. collect_summary: G0/G1 plus a G2 copy in the gVCF reads G1/G2", - apol1["g2"]["result"] == "G1/G2" and apol1["g2"]["source"] == "vcf/S.g.vcf.gz", apol1.get("g2")) - check("only the APOL1 line carries g2", others == [], others) - page = "\n".join(rr.rules_html({"rules": rows}, "S")) - check("render_report HTML: With G2: G1/G2 and the high-risk reading", - "With G2: G1/G2" in page and "Two APOL1 risk copies" in page, page[:400]) - text = "\n".join(rr.traits_text({"called": 0, "not_covered": 0, "other_allele": 0, "rows": [], - "rules": rows}, "S")) - check("render_report text: With G2 (rs71785313): G1/G2", "With G2 (rs71785313): G1/G2" in text, text) - d = folder("ref", [BLOCK], "gvcf") - apol1 = next(r for r in cs.genotype_rules(d, "S") if r["rule"] == "APOL1") - check("control: a reference gVCF gives G0/G1", apol1["g2"]["result"] == "G0/G1", apol1.get("g2")) - d = folder("vcf", [], "vcf") - apol1 = next(r for r in cs.genotype_rules(d, "S") if r["rule"] == "APOL1") - check("a VCF with no G2 record: G0/G1 (no G2 call), from vcf/S.vcf.gz", - apol1["g2"]["result"] == "G0/G1 (no G2 call)" and apol1["g2"]["source"] == "vcf/S.vcf.gz", - apol1.get("g2")) - d = os.path.join(work, "none") - os.makedirs(os.path.join(d, "traits")) - shutil.copy(os.path.join(work, "het", "traits", "S_genotype_rules.tsv"), os.path.join(d, "traits")) - apol1 = next(r for r in cs.genotype_rules(d, "S") if r["rule"] == "APOL1") - check("no VCF at all: G2 not read", apol1["g2"]["status"] == "not_read" and apol1["g2"]["result"] == ".", - apol1.get("g2")) - tb = os.path.join(work, "tbi", "vcf", "S.g.vcf.gz") - os.makedirs(os.path.dirname(tb)) - pad = [rec(36000000 + i * 1000, "A", "<*>", "0/0", ".", f"END={36000000 + i * 1000 + 999}") for i in range(300)] - lines = [head] + ["\t".join(r) + "\n" for r in sorted(pad + [rec(*LEFT, "1/1")], key=lambda r: int(r[1]))] - if bgzip_with_tbi(tb, lines): - got = m.read_g2(cs.vcf_window(tb, "chr22", m.CONTEXT_START, m.SPAN[1]), "gvcf") - check("vcf_window through the .tbi finds the G2 record (2 copies)", got["copies"] == 2, got) - else: - print("[SKIP] no bgzip and tabix here: the .tbi path runs where they are installed") - finally: - shutil.rmtree(work) - - -def main(): - marker_checks() - read_checks() - combine_checks() - report_checks() - print(f"\n{'FAILED' if FAILS else 'ALL PASSED'}: {FAILS} failure(s)") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_hla_report.py b/tests/test_hla_report.py index aef51e80b..94e1f3812 100644 --- a/tests/test_hla_report.py +++ b/tests/test_hla_report.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """The HLA section of the reports (step 08): bin/hla_alleles.tsv, what -bin/collect_summary.py derives from T1K's genotype TSV, and what +og summary collect derives from T1K's genotype TSV, and what bin/render_report.py prints. Synthetic T1K rows only. Checks: @@ -103,6 +103,11 @@ def t1k(rows): return "".join(out) +# The label og summary collect gives T1K's HFE row. +HLA_HFE_LABEL = ("T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the " + "variant calls (ClinVar section)") + + def summary(work, name, rows): d = os.path.join(work, name, "S") os.makedirs(os.path.join(d, "hla_t1k"), exist_ok=True) @@ -118,7 +123,8 @@ def main(): # 1. the table errs = table_errors(TABLE) check("bin/hla_alleles.tsv passes its schema and lists every required allele", not errs, errs) - check("collect_summary reads the same rows", [r["allele"] for r in cs.hla_read_table()] + derived = summary(work, "table", [("HLA-B", "B*57:01:01", 40, "B*08:01:01", 40)])["sections"]["hla"]["data"] + check("og summary collect reads the same rows", [r["allele"] for r in derived["derived"]["alleles"]] == [l.split("\t")[0] for l in open(TABLE) if not l.startswith("#") and l.strip()][1:]) good = open(TABLE).read() for desc, bad in (("a wrong kind", good.replace("\tdrug\t", "\tdrugs\t", 1)), @@ -205,7 +211,7 @@ def state(summ, allele): check("a quality-0 allele with 12 candidates is stored with three and the count", loci["HLA-DRB5"]["alleles"] == ["DRB5*01:01,DRB5*01:02,DRB5*01:03"] and loci["HLA-DRB5"]["candidates"] == [12], loci["HLA-DRB5"]) - check("the HFE row has its label", loci["HLA-HFE"].get("label") == cs.HLA_HFE_LABEL, loci["HLA-HFE"]) + check("the HFE row has its label", loci["HLA-HFE"].get("label") == HLA_HFE_LABEL, loci["HLA-HFE"]) check("cap summary validates", not validate.validate(schema, json.loads(json.dumps(cap))), validate.validate(schema, json.loads(json.dumps(cap)))) txt, html = render_report.text_report(cap), render_report.html_report(cap) diff --git a/tests/test_locityper_report.py b/tests/test_locityper_report.py index 5bbae64a9..122feb26a 100755 --- a/tests/test_locityper_report.py +++ b/tests/test_locityper_report.py @@ -429,13 +429,8 @@ def og_cases(t): cyp = [x for x in s["sections"]["locityper"]["data"]["loci"] if x["locus"] == "CYP2D6"][0] check("5. a primary call older than the last run says it is stale", "(stale: from an earlier run)" in cyp["primary"], (s["sections"]["cyrius"]["state"], cyp)) - hla = {"loci": [{"gene": "DRB1", "alleles": ["*15:01", "*03:01"], "low_confidence": True}, - {"gene": "DQB1", "alleles": ["*06:02", "*02:01"], "low_confidence": False}]} - prim = collect_summary.locityper_primary({"hla": {"state": "ok", "data": hla}, "pypgx": {"state": "missing"}, - "cyrius": {"state": "missing"}}) - check("5. a low-confidence T1K call says so in the primary text, a confident one does not", - prim["HLA-DRB1"] == "T1K (step 08): *15:01/*03:01 (low confidence)" - and prim["HLA-DQB1"] == "T1K (step 08): *06:02/*02:01", prim) + # A low-confidence T1K call in the primary text: og-summary's own test + # (collect::group_c::tests, the Locityper primary calls). os.makedirs(os.path.join(t, "none", "S")) s = collect_summary.collect("S", os.path.join(t, "none", "S")) check("5. without step 53 there is no card and no text section", diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index 03ab6f15f..ae634adb0 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -52,6 +52,7 @@ Run: OG=rust/target/release/og python3 tests/test_lpa_kiv2.py """ +import csv import math import os import subprocess @@ -75,6 +76,11 @@ from test_copy_number import cn # noqa: E402 pinned differential oracle FAILS = 0 +# The seven copies of KIV-2 4925G>A on GRCh38 (rust/crates/og-cn/src/markers.rs LPA_KIV2_4925). +KIV2_4925_COPIES = (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932) +# The columns of step 42's traits table (og traits table --out). +TRAITS_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", + "note", "pmids"] # The two LPA rows; named once here so no line of this file reads as an rsID with a genotype. R1, R2 = "rs10455872", "rs3798220" ROW = cn.load_catalog(CATALOG)["LPA_KIV2"] @@ -153,11 +159,10 @@ def cv(vals): def pileup(path, per_site, extra=""): - """mpileup lines at the seven copies of 4925G>A (traits_report.MARKERS); + """mpileup lines at the seven copies of 4925G>A (KIV2_4925_COPIES); per_site is (C reads, T reads) at each.""" - import traits_report with open(path, "w") as f: - for p in traits_report.MARKERS["LPA_KIV2_4925"]["copies"]: + for p in KIV2_4925_COPIES: c, t_ = per_site bases = "^]" + "C" * c + "t" * t_ + "$" f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") @@ -166,10 +171,11 @@ def pileup(path, per_site, extra=""): def traits(): """6. assets/traits.tsv holds the two LPA risk variants the report counts.""" - import collect_summary - import traits_report - table = {r["rsid"]: r for r in traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv"))} - for rs, alt in collect_summary.LPA_RISK_ALLELES.items(): + with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: + table = {r["rsid"]: r for r in csv.DictReader((line for line in f if line.strip() and not line.startswith("#")), + delimiter="\t", quoting=csv.QUOTE_NONE)} + # The two risk alleles og summary collect counts for the Lp(a) flag. + for rs, alt in {"rs10455872": "G", "rs3798220": "C"}.items(): r = table.get(rs, {}) check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", r.get("gene") == "LPA" and r.get("alt") == alt, r) @@ -277,7 +283,6 @@ def main(): traits() import collect_summary import render_report - import traits_report # 7. reports def sample(t, name, n_copies, flank=30.0, genotypes=None): @@ -291,11 +296,11 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): if genotypes: os.makedirs(os.path.join(d, "traits")) with open(os.path.join(d, "traits", "S_traits.tsv"), "w") as f: - f.write("\t".join(traits_report.OUT_COLUMNS) + "\n") + f.write("\t".join(TRAITS_COLUMNS) + "\n") for rs, gt in genotypes.items(): - row = {k: "." for k in traits_report.OUT_COLUMNS} + row = {k: "." for k in TRAITS_COLUMNS} row.update(rsid=rs, gene="LPA", genotype=gt, status="called" if gt != "." else "not_covered") - f.write("\t".join(row[k] for k in traits_report.OUT_COLUMNS) + "\n") + f.write("\t".join(row[k] for k in TRAITS_COLUMNS) + "\n") summ = collect_summary.collect("S", d) return summ, render_report.text_report(summ), render_report.html_report(summ) diff --git a/tests/test_mei_report.py b/tests/test_mei_report.py index d87f1c5cb..34873cdc9 100644 --- a/tests/test_mei_report.py +++ b/tests/test_mei_report.py @@ -179,20 +179,37 @@ def status(result, gene): # 5. the summary and the reports import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 + +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + +MEI_NOT_ASSESSED = ("Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): " + "not looked for (step 50, MEGAnE, is opt-in)") + with tempfile.TemporaryDirectory() as d: os.makedirs(f"{d}/S") summ = cs.collect("S", f"{d}/S") check("5 without step 50 the not-assessed list names mobile element insertions", - cs.MEI_NOT_ASSESSED in summ["not_assessed"] and cs.MEI_NOT_ASSESSED.startswith("Mobile element insertions"), + MEI_NOT_ASSESSED in summ["not_assessed"] and MEI_NOT_ASSESSED.startswith("Mobile element insertions"), True) - check("5 no table: no section", cs.sec_mei(f"{d}/S", "S"), (None, {})) + check("5 no table: no section", sec(f"{d}/S", "S", "mei"), (None, {})) # A step 50 that ran and failed was looked for: it is listed as failed, not as opt-in. os.makedirs(f"{d}/F/logs") with open(f"{d}/F/logs/run_status.tsv", "w") as f: f.write(f"meta\tstarted_epoch\t{time.time() - 60}\nstep\t50\tfailed (exit 1)\n") summ = cs.collect("F", f"{d}/F") check("5 a failed step 50 is listed under not run, not as not looked for", - (summ["sections"]["mei"]["state"], cs.MEI_NOT_ASSESSED in summ["not_assessed"], + (summ["sections"]["mei"]["state"], MEI_NOT_ASSESSED in summ["not_assessed"], any("step 50 failed" in x for x in summ["not_run"])), ("failed", False, True)) os.makedirs(f"{d}/S/mei") for name, text in (("m.vcf", HEAD + FKTN_SVA + rec("chr20", 5040478, "SINE/Alu")), ("sites.tsv", SITES), @@ -202,14 +219,14 @@ def status(result, gene): subprocess.run([OG, "mei", "report", "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", f"{d}/sites.tsv", "--depth", f"{d}/depth.tsv", "--out-calls", f"{d}/S/mei/S_mei.tsv", "--out-sites", f"{d}/S/mei/S_mei_sites.tsv"], check=True) - path, data = cs.sec_mei(f"{d}/S", "S") + path, data = sec(f"{d}/S", "S", "mei") check("5 the summary finds the table", path.endswith("mei/S_mei_sites.tsv"), True) check("5 the summary counts the calls per element", (data["calls"], data["by_element"]), (2, {"Alu": 1, "SVA": 1})) check("5 the summary has each catalog row", [(x["gene"], x["status"]) for x in data["sites"]], [("FKTN", "seen"), ("TAF1", "not seen")]) summ = cs.collect("S", f"{d}/S") check("5 with step 50's tables, mobile element insertions are not listed as not assessed", - cs.MEI_NOT_ASSESSED in summ["not_assessed"], False) + MEI_NOT_ASSESSED in summ["not_assessed"], False) try: page = rr.html_report(summ) except Exception as e: # the whole step 24 page is lost if the card raises @@ -226,23 +243,23 @@ def status(result, gene): good_calls = open(calls_path).read() with open(calls_path, "w") as f: f.write(good_calls.splitlines()[0] + "\n") - check("5 a calls table with its header and no row is zero calls", cs.sec_mei(f"{d}/S", "S")[1]["calls"], 0) + check("5 a calls table with its header and no row is zero calls", sec(f"{d}/S", "S", "mei")[1]["calls"], 0) for name, text in (("an empty calls table", ""), ("a calls table without its columns", "chrom\tpos\n")): with open(calls_path, "w") as f: f.write(text) try: - cs.sec_mei(f"{d}/S", "S") + sec(f"{d}/S", "S", "mei") check(f"5 {name} is unreadable", "read", "Unreadable") - except cs.Unreadable: + except Unreadable: check(f"5 {name} is unreadable", "Unreadable", "Unreadable") with open(calls_path, "w") as f: f.write(good_calls) with open(f"{d}/S/mei/S_mei_sites.tsv", "w") as f: f.write("gene\tstatus\nFKTN\tseen\n") try: - cs.sec_mei(f"{d}/S", "S") + sec(f"{d}/S", "S", "mei") check("5 a sites table without its columns is unreadable", "read", "Unreadable") - except cs.Unreadable: + except Unreadable: check("5 a sites table without its columns is unreadable", "Unreadable", "Unreadable") # 6. the real catalog diff --git a/tests/test_mito_extended.py b/tests/test_mito_extended.py index a409b153e..47304a04a 100644 --- a/tests/test_mito_extended.py +++ b/tests/test_mito_extended.py @@ -215,8 +215,7 @@ def sample_dir(tmp, name, records, deletions=None, idxstats=None): def mito_section(d, name): import collect_summary - _, data = collect_summary.sec_mito(d, name) - return data + return collect_summary.collect(name, d)["sections"]["mito"]["data"] def reports(d, name): @@ -247,13 +246,11 @@ def test_confirmed_snv(tmp): "Confirmed disease variants (assets/mito_confirmed.tsv): 0" in text) import collect_summary saved, collect_summary.ASSETS_DIR = collect_summary.ASSETS_DIR, os.path.join(tmp, "no-assets") - collect_summary._ASSETS.clear() try: data = mito_section(d, "Snone") text, html = reports(d, "Snone") finally: collect_summary.ASSETS_DIR = saved - collect_summary._ASSETS.clear() check("without the list the summary has no confirmed count, only why", "confirmed" not in data and "mito_confirmed.tsv is not in" in (data.get("confirmed_note") or ""), str(data)) check("without the list neither report shows a count of confirmed variants", diff --git a/tests/test_mocha_report.py b/tests/test_mocha_report.py index 55a94a56a..80aa58c86 100644 --- a/tests/test_mocha_report.py +++ b/tests/test_mocha_report.py @@ -48,6 +48,23 @@ import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + +MCA_NOT_ASSESSED = ("Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share " + "of the blood cells): step 51 (MoChA) did not assess this sample") + + FAILS = [] STATS_HEAD = ("sample_id\tcomputed_gender\tcall_rate\tcov_median\tcov_sd\tcov_auto\tbaf_corr\tbaf_conc\tbaf_auto" "\tn_sites\tn_hets\n") @@ -148,7 +165,7 @@ def cases_1_to_6(): # 6. both reports _, _, _, d = run([call(), call(chrom="chrX")]) - path, data = cs.sec_mocha(f"{d}/S", "S") + path, data = sec(f"{d}/S", "S", "mosaic") check("6 the section lists the reported call only", [(x["type"], x["chrom"], x["cell_fraction"]) for x in data["calls"]], [("CN-LOH", "chr20", "0.150")]) check("6 and counts the other", data["not_reported"], 1) @@ -157,24 +174,24 @@ def cases_1_to_6(): "about 0.150 of the cells") check("6 the first line gives the floor", lines[0].startswith("Reported at a cell fraction of 0.050 or more at 30.0x"), True) summ = cs.collect("S", f"{d}/S") - check("6 assessed: no not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], False) + check("6 assessed: no not-assessed line", MCA_NOT_ASSESSED in summ["not_assessed"], False) os.remove(f"{d}/S/mosaic/S_mocha_summary.tsv") summ = cs.collect("S", f"{d}/S") - check("6 without step 51: the not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], True) + check("6 without step 51: the not-assessed line", MCA_NOT_ASSESSED in summ["not_assessed"], True) _, _, _, d = run([call()], baf_auto=0.05) - data = cs.sec_mocha(f"{d}/S", "S")[1] + data = sec(f"{d}/S", "S", "mosaic")[1] check("6 not assessed: the text says why", rr.mosaic_mca_lines(data)[0].startswith("Not assessed: phased BAF"), True) - check("6 and the not-assessed line stays", cs.MCA_NOT_ASSESSED in cs.collect("S", f"{d}/S")["not_assessed"], True) + check("6 and the not-assessed line stays", MCA_NOT_ASSESSED in cs.collect("S", f"{d}/S")["not_assessed"], True) check("6 no reported call: the text says none is reported, not that none exists", - rr.mosaic_mca_lines(cs.sec_mocha(f"{run([call(cf=0.04)])[3]}/S", "S")[1])[1].startswith("No mosaic alteration reported"), True) + rr.mosaic_mca_lines(sec(f"{run([call(cf=0.04)])[3]}/S", "S", "mosaic")[1])[1].startswith("No mosaic alteration reported"), True) check("6 the pipeline-wide not-assessed list keeps large mosaic alterations out of its chrM-only line", - [x for x in cs.NOT_ASSESSED if x.startswith("Mosaic")], ["Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)"]) + [x for x in cs.collect("S", f"{d}/S")["not_assessed"] if x.startswith("Mosaic")], ["Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)"]) def unreadable(d): try: - cs.sec_mocha(f"{d}/S", "S") - except cs.Unreadable as e: + sec(f"{d}/S", "S", "mosaic") + except Unreadable as e: return str(e) return None diff --git a/tests/test_paralog_loci.py b/tests/test_paralog_loci.py index c6fc5a529..58c9c2ac0 100755 --- a/tests/test_paralog_loci.py +++ b/tests/test_paralog_loci.py @@ -26,7 +26,7 @@ genes it does not list are the 17 of NOT_IN_CALLABILITY; 4. the report readers: the not-assessed row step 35 writes (its own lines, cut from scripts/35-paralogs.sh and run) gives a reading that names the - reason in bin/collect_summary.py, and in bin/paraphase_report.py and + reason in the summary (og summary collect), and in bin/paraphase_report.py and bin/couple_report.py (the Paraphase side-by-side and the couple screen), and a called table still reads as before. Control: the old row with "." for its place is not found over SMN1 exon 7 there. A catalog table @@ -262,6 +262,15 @@ def unlisted(cal_rows): # 4. the report reader on step 35's tables sys.path.insert(0, os.path.join(REPO, "bin")) import collect_summary # noqa: E402 + + def parascopy(d): + """Step 35's section of the summary og summary collect writes for folder D.""" + return collect_summary.collect("S", d)["sections"]["parascopy"]["data"] + + def gba1(d): + """The gba1 catalog row of that section.""" + return next(x for x in parascopy(d)["loci"] if x["table"] == "gba1") + head11 = ("chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" "homologous_regions\n") with tempfile.TemporaryDirectory() as d: @@ -279,7 +288,7 @@ def unlisted(cal_rows): os.environ, region="chr5:70951946-70951946", model="SMN1.gz", reason=reason, out=t)) import paraphase_report # noqa: E402 import couple_report # noqa: E402 - _, data = collect_summary.sec_parascopy(d, "S") + data = parascopy(d) check("report: a not-assessed SMN table reads as not assessed, with the reason", data["reading"] == f"SMN1 and SMN2 not assessed: {reason}: no carrier reading" and data["smn1"]["cn"] is None, data.get("reading")) @@ -296,7 +305,7 @@ def unlisted(cal_rows): with open(t, "w") as f: f.write(head11 + "chr5\t70925000\t70953000\tSMN1\tPASS\t4\t40\tPASS\t1,3\t35,30\t" "chr5:70049000-70077000:-\n") - _, data = collect_summary.sec_parascopy(d, "S") + data = parascopy(d) check("report: a called SMN table still reads one SMN1 copy as a carrier", data["smn1"]["cn"] == 1 and "carrier" in data["reading"] and "not assessed" not in data["reading"], data.get("reading")) @@ -310,12 +319,12 @@ def unlisted(cal_rows): "homologous_regions": "*"} with open(g, "w") as f: f.write("\t".join(cols) + "\n" + "\t".join(vals[c] for c in cols) + "\n") - loc = collect_summary.parascopy_locus("gba1", g, None) + loc = gba1(d) check(f"report: a catalog table without {drop} is unreadable, not 'not resolved'", loc["status"] == "unreadable" and loc["cn"] is None, loc) with open(g, "w") as f: f.write(head11 + "chr1\t155231478\t155244699\tGBA\tPASS\t4\t40\tPASS\t2,2\t30,30\t*\n") - loc = collect_summary.parascopy_locus("gba1", g, None) + loc = gba1(d) check("control: the same table with every column is called", loc["status"] == "called" and loc["cn"] == 4, loc) if FAILS: diff --git a/tests/test_pypgx_genes.sh b/tests/test_pypgx_genes.sh index 18810d0dc..a1273b452 100755 --- a/tests/test_pypgx_genes.sh +++ b/tests/test_pypgx_genes.sh @@ -241,7 +241,7 @@ def problems(lines): os.makedirs(os.path.join(d, "pypgx")) with open(os.path.join(d, "pypgx", "s1_pypgx_summary.tsv"), "w") as f: f.write("\n".join("\t".join(r) for r in [hdr] + lines) + "\n") - _, data = collect_summary.sec_pypgx(d, "s1") + data = collect_summary.collect("s1", d)["sections"]["pypgx"]["data"] # The rows of the block (gene, text), with CNVpytor's section beside pypgx's: # pypgx owns every bam gene, so each has one row, and it is pypgx's. block = render_report.pgx_copy_number({"pypgx": {"state": "ok", "data": data}, diff --git a/tests/test_render_report.py b/tests/test_render_report.py index beb0497c0..1d85a1758 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -133,6 +133,21 @@ def check(desc, ok, detail=""): FAILS += 1 +PRS_NOT_ADJUSTED = "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums" + + +def haplogroup_explore(kind, hg): + """The explore links og summary collect writes for one Y ('y') or mitochondrial ('mt') haplogroup call.""" + with tempfile.TemporaryDirectory() as d: + if kind == "y": + put(f"{d}/y_haplogroup/S_y_haplogroup.txt", f"Sample_name\tHg\tValid_markers\tQC-score\nS\t{hg}\t9\t1\n") + key = "y_haplogroup" + else: + put(f"{d}/mito/S_haplogroup.txt", f"SampleID\tHaplogroup\nS\t{hg}\n") + key = "haplogroup" + return collect_summary.collect("S", d)["sections"][key]["data"]["explore"] + + def put(path, text, gz=False): os.makedirs(os.path.dirname(path), exist_ok=True) with (gzip.open(path, "wt") if gz else open(path, "w")) as f: @@ -330,7 +345,7 @@ def chapter_checks(work): ("missing", "missing"), ("ok", "failed"))]) for kind, hg in (("y", "R-M269*(xR-L23)"), ("mt", "H1a+16189")): - rows = render_report.explore_html({"haplogroup": hg, "explore": collect_summary.haplogroup_explore(kind, hg)}) + rows = render_report.explore_html({"haplogroup": hg, "explore": haplogroup_explore(kind, hg)}) marks = re.findall(r'\(([^)]*)\)', "".join(rows)) want = [render_report.OFFSITE_HAPLOGROUP] * (len(marks) - 1) + [render_report.OFFSITE] check(f"chapters: {hg}: the FamilyTreeDNA and YFull links say the address carries the haplogroup, the docs link does not", @@ -740,7 +755,7 @@ def chrx_of(res): and "PGS000002 (3)" not in txt, txt) check("no panel: the HTML row says raw score only", "" in html) check("no panel: not-assessed names the missing percentiles", - collect_summary.PRS_NOT_ADJUSTED in summ["not_assessed"]) + PRS_NOT_ADJUSTED in summ["not_assessed"]) d = f"{work}/sample_adj/S" put(f"{d}/prs/S_prs_summary.tsv", "\t".join(PRS_COLUMNS) + "\n" "Coronary artery disease\tPGS000001\t1.25\t2\t3\t66.7\t94.4\tEUR\t0\tgvcf\n") @@ -752,7 +767,7 @@ def chrx_of(res): check("panel: the note names the group and the panel, no raw-score line, no chrX line", "among the EUR samples of the pgsc_1000G_v1 reference panel" in txt and "Raw score only" not in txt and "Raw score only" not in html and "Rows on chrX" not in txt, txt) - check("panel: no not-assessed line about percentiles", collect_summary.PRS_NOT_ADJUSTED not in summ["not_assessed"]) + check("panel: no not-assessed line about percentiles", PRS_NOT_ADJUSTED not in summ["not_assessed"]) caveat = ("Most PRS scores were developed in European-ancestry populations and predict less well for " "other ancestries, even with an ancestry-adjusted percentile.") check("panel: the ancestry-transfer caveat in both reports", caveat in txt and caveat in html, txt) diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index 075e36827..23dc05b2d 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -30,7 +30,7 @@ scripts/24-html-report.sh) pass it with _charts.json beside the report, and two renders of the demo give the same file apart from generated_utc; - 9. run_delta.py sees no change between a summary without + 9. og run delta sees no change between a summary without heteroplasmic_sites and one with it; 10. the live page (bin/owngenome_live.py with bin/owngenome_live_charts.py and assets/live/plugins/charts.js): the plugin serves _charts.json and @@ -48,15 +48,16 @@ import os import re import shutil +import subprocess import sys import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 import render_report # noqa: E402 import report_charts # noqa: E402 -import run_delta # noqa: E402 + +OG = os.environ.get("OG") or os.path.join(REPO, "bin", "og") FAILS = 0 GREENS = ("#d5f5e3", "#1d3b2a", "#8ee0ae", "#196f3d", "green") @@ -278,8 +279,8 @@ def unit_checks(): nd.get("ClinVar hits on chrM") == 1 and nd.get("ClinVar hits on other contigs (alt, unplaced)") == 1 and "Not drawn:" in g["svg"] and not any(m["chrom"] in ("chrM", "chr22_KI270879v1_alt") for m in g["marks"]), nd) - check("4. genome: the 5 MB floor is collect_summary.ROH_MIN_BP", - report_charts.ROH_MIN_MB * 1e6 == collect_summary.ROH_MIN_BP) + # The floor of the roh section (og-summary, collect/group_c/roh.rs ROH_MIN_BP). + check("4. genome: the 5 MB floor is the roh section's", report_charts.ROH_MIN_MB * 1e6 == 5e6) for region in ("chr3:10000000_22300000", "3:1-2", "chr3:20-10"): bad = copy.deepcopy(s) bad["sections"]["roh"]["data"]["autosomal_over_5mb"][0]["region"] = region @@ -426,14 +427,31 @@ def demo_checks(work): check("7. the hits table carries the id the charts link to, and its rows keep their shape", page.count('') == 1 and len(re.findall(r"" in html, True) check("7 the card lists the low call too", "" in html, True) @@ -328,9 +342,9 @@ def good(gb="0", dp="20"): with open(os.path.join(sd, "y_str", "S1_y_str.tsv"), "w") as f: f.write(t) try: - cs.sec_y_str(sd, "S1") + sec(sd, "S1", "y_str") check("7 an unknown confidence is unreadable", "read", "Unreadable") -except cs.Unreadable: +except Unreadable: check("7 an unknown confidence is unreadable", "Unreadable", "Unreadable") try: page = rr.html_report(cs.collect("S1", sd)) From 2545b38bf3ba63ef2b1027f9f88f11a0f40f62a9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:32:53 +0100 Subject: [PATCH 22/86] refactor(summary): delete the Python the summary switch left unused With summary.json written by og summary collect, nothing imports pgx_outside_calls.py, ancestry_fine_report.py, clip_check.py, annotsv_postfilter.py, chip_report.py, traits_report.py or genotype_rules_indels.py any more. They go, with test_clip_check.py and test_genotype_rules_indels.py (their checks are og's cargo tests now). The e2e cases that read a pinned record from them read og-summary's and og-cn's constants. collect_summary.py is listed as a remnant for the report, couple, index and live code until their own ports. Bead: pgp-bs9u.45 --- .github/workflows/guard.yml | 2 +- assets/genotype_rules.tsv | 2 +- assets/imprinted_regions.tsv | 4 +- bin/render_report.py | 2 +- scripts/ci/e2e-case-paths.tsv | 186 +++++++++++----------- tests/e2e/genotype-rules-2-g2.sh | 35 ++-- tests/e2e/genotype-rules-2-lpa-null.sh | 14 +- tests/e2e/genotype-rules-2-smn1-marker.sh | 14 +- tests/e2e/str-extra-repeats-1-readings.sh | 6 +- tests/parity/remnants.tsv | 6 +- tests/test_ancestry_fine_report.py | 68 +------- tests/test_e2e_scope.sh | 18 +-- tests/test_pgx_outside_calls.py | 32 +--- 13 files changed, 160 insertions(+), 229 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 4dfdc9daf..7f6ae2a0b 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -624,7 +624,7 @@ jobs: tests/test_results_index.py \ tests/test_pgx_outside_calls.py tests/test_hla_second_caller.py tests/test_paraphase_report.py \ tests/test_copy_number.py tests/test_hba_copy_number.py \ - tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_genotype_rules_indels.py tests/test_callability.py; do + tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_callability.py; do rc=0 echo "::group::${t}" case "$t" in diff --git a/assets/genotype_rules.tsv b/assets/genotype_rules.tsv index fb8eb55cf..bd99e77bf 100644 --- a/assets/genotype_rules.tsv +++ b/assets/genotype_rules.tsv @@ -41,7 +41,7 @@ haplotype TAS2R38 AVI C,A,T combination TAS2R38 PAV/PAV PAV/PAV Two copies of PAV, the taster form: in Bufe 2005 this group rated PTC and PROP the most bitter. combination TAS2R38 PAV/AVI PAV/AVI One PAV and one AVI copy: a taster. In Bufe 2005 this group recognised PTC and PROP as bitter at concentrations close to those of PAV/PAV; how bitter they rated them lay between the two homozygous groups and varied widely. combination TAS2R38 AVI/AVI AVI/AVI Two copies of AVI, the non-taster form: in Bufe 2005 the AVI receptor did not respond to PTC, and this group recognised PTC as bitter only at far higher concentrations. PROP responses varied more. -rule APOL1 APOL1 APOL1 kidney risk variants (G1 read, G2 not read) no yes G1 is two missense changes that sit on one copy, S342G (rs73885319) and I384M (rs60910145), in perfect linkage disequilibrium in Genovese 2010, so they are read together without phase. G2 (rs71785313) is a 6-base deletion of N388 and Y389; this rule table reads single-base sites only, so this rule does not read G2 and a G2 copy reads as G0 here; the HTML report reads G2 from the sample's VCF beside this result (bin/genotype_rules_indels.py). G1 and G2 never share a copy. In African Americans, two risk copies (G1 or G2) went with focal segmental glomerulosclerosis (odds ratio 10.5) and hypertension-attributed end-stage kidney disease (odds ratio 7.3), and one copy did not raise the risk (a recessive pattern). Both variants are common on African chromosomes and were absent from the European chromosomes studied. 20647424 +rule APOL1 APOL1 APOL1 kidney risk variants (G1 read, G2 not read) no yes G1 is two missense changes that sit on one copy, S342G (rs73885319) and I384M (rs60910145), in perfect linkage disequilibrium in Genovese 2010, so they are read together without phase. G2 (rs71785313) is a 6-base deletion of N388 and Y389; this rule table reads single-base sites only, so this rule does not read G2 and a G2 copy reads as G0 here; the HTML report reads G2 from the sample's VCF beside this result (og summary collect). G1 and G2 never share a copy. In African Americans, two risk copies (G1 or G2) went with focal segmental glomerulosclerosis (odds ratio 10.5) and hypertension-attributed end-stage kidney disease (odds ratio 7.3), and one copy did not raise the risk (a recessive pattern). Both variants are common on African chromosomes and were absent from the European chromosomes studied. 20647424 site APOL1 rs73885319 chr22 36265860 A G site APOL1 rs60910145 chr22 36265988 T G haplotype APOL1 G0 A,T diff --git a/assets/imprinted_regions.tsv b/assets/imprinted_regions.tsv index b83ec1c89..c71bb24dd 100644 --- a/assets/imprinted_regions.tsv +++ b/assets/imprinted_regions.tsv @@ -1,6 +1,6 @@ # Imprinting disorders caused by uniparental disomy (UPD), one row per chromosome or region. -# Read by bin/collect_summary.py (IMPRINTED_REGIONS holds the same rows; tests/test_collect_summary.py -# checks that the two agree) to name what a run of homozygosity over most of one chromosome could mean. +# Read by og summary collect (IMPRINTED_REGIONS in rust/crates/og-summary holds the same rows; its +# tests check that the two agree) to name what a run of homozygosity over most of one chromosome could mean. # start and end: GRCh38, 0-based start, exclusive end, the UCSC cytoBand.hg38.txt bands of the region; # empty for a row that applies to the whole chromosome. A row with a region applies when a run on the # flagged chromosome overlaps it. "-" is no known imprinting disorder for UPD from that parent. diff --git a/bin/render_report.py b/bin/render_report.py index eac22b4f4..e949a759b 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -404,7 +404,7 @@ def cyp2d6_verdict(c): def probable_text(p): - """The probable CYP2D6 call (pgx_outside_calls.probable_cyp2d6) as one line.""" + """The probable CYP2D6 call (og summary collect's cyp2d6 block) as one line.""" return (f"{p.get('diplotype')}" + (f", {p['phenotype']}" if p.get("phenotype") else "") + "; not passed to PharmCAT, so no drug guidance rests on it") diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 7f5b093d3..50dd13368 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -40,143 +40,143 @@ 51-text-report - 60-nextflow - 90-report-assets - -acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh -alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh +alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh -ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/ancestry_fine_report.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh -annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh -annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh +annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh +annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh bash-step-08-hla - bash-step-09-str assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh bash-step-10-telomere scripts/10-telomere-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bash-step-11-roh scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh bash-step-14-imputation scripts/14-imputation-prep.sh scripts/ci/fake-docker/lib.sh bash-step-19-delly scripts/19-delly.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh -bash-step-bash42 assets/construct_variants.tsv bin/clip_check.py bin/cpsr_sample_id bin/og scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh +bash-step-bash42 assets/construct_variants.tsv bin/cpsr_sample_id bin/og scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh bloodtype-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh -callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +chip-clonal-1 assets/chip_genes.tsv bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/og scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz ci-e2e-2-locityper assets/locityper_loci.bed bin/og scripts/53-locityper.sh scripts/setup.sh -clinical-filter-mane-and-quality-1-tiers assets/construct_variants.tsv bin/clip_check.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh +clinical-filter-mane-and-quality-1-tiers assets/construct_variants.tsv bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh clinical-filter-mane-and-quality-2-vep-norm scripts/13-vep-annotation.sh scripts/ci/fake-docker/lib.sh -clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinical_filter/* modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf -clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-guard-1-mapq0 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinical_filter/* modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh +clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf +clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh +cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-guard-1-mapq0 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh -cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs -construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv -family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh -followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh -followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh -followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og +followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh +followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh +followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og genome-lookup-1 bin/og scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv bin/traits_report.py modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/genotype_rules_indels.py bin/traits_report.py -genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh -genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh -haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py +genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh +genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh +haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/og scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh -mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og bin/traits_report.py scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh -nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh -nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/traits_report.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/clip_check.py bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py bin/traits_report.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz +lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh +mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh +nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/cnvpytor_to_vcf.py bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* -paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz pgx-1-depth-check bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh -pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh pharmcat-indel-refblocks-1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh tests/fixtures/pharmcat/refblocks.g.vcf pharmcat-mito-input-and-log-1-mtrnr1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh platelet-antigens-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh -prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py bin/traits_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -qc-1-sample-qc bin/collect_summary.py bin/og bin/traits_report.py scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -qc-3-cram assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh +prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-1-sample-qc bin/collect_summary.py bin/og scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-3-cram assets/callability_genes.tsv bin/collect_summary.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/clip_check.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh -reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/traits_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh +reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh reports-and-pgx-correctness-revel scripts/30-vcfanno.sh scripts/ci/fake-docker/lib.sh -run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh run-all-sample-qc-and-stale-index-1-index-times assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in -sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh sv-mito-telomere-steps-4-clinvar-tier bin/og scripts/23-clinical-filter.sh scripts/ci/fake-docker/lib.sh sv-mito-telomere-steps-5-giab scripts/benchmark-variants.sh scripts/ci/fake-docker/lib.sh sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b-freebayes.sh scripts/ci/fake-docker/lib.sh -traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/og bin/traits_report.py modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh diff --git a/tests/e2e/genotype-rules-2-g2.sh b/tests/e2e/genotype-rules-2-g2.sh index 5932e63c3..ee560b54b 100644 --- a/tests/e2e/genotype-rules-2-g2.sh +++ b/tests/e2e/genotype-rules-2-g2.sh @@ -1,20 +1,23 @@ #!/usr/bin/env bash -# APOL1 G2 (rs71785313) beside step 42's APOL1 rule (bin/genotype_rules_indels.py), +# APOL1 G2 (rs71785313) beside step 42's APOL1 rule (og summary collect's G2 +# reader, the apol1 module of rust/crates/og-summary/src/collect/group_c/traits.rs), # with the real bcftools images: -# - dbSNP orientation, online: the record pinned in the module (DBSNP) is what -# dbSNP's GRCh38 VCF of the build versions.env pins holds for rs71785313 (same -# position and REF, the module's ALT among dbSNP's); control: the same compare on -# a copy with REF and ALT swapped fails; +# - dbSNP orientation, online: the record pinned in that module (POS, REF, ALT) is +# what dbSNP's GRCh38 VCF of the build versions.env pins holds for rs71785313 +# (same position and REF, the module's ALT among dbSNP's); control: the same +# compare on a copy with REF and ALT swapped fails; # - the report's reader through a real .tbi: a bgzipped, indexed gVCF (bcftools -# view -Oz, bcftools index -t) with a G2 deletion, read by collect_summary's -# genotype_rules, gives G1/G2 beside a G0/G1 rule line; control: the same -# folder with a reference block over G2 gives G0/G1. +# view -Oz, bcftools index -t) with a G2 deletion, read by og summary collect, +# gives G1/G2 beside a G0/G1 rule line; control: the same folder with a +# reference block over G2 gives G0/G1. # Writes one folder under GENOME_DIR, removed at the end; no other case reads it. . "$(dirname "$0")/lib.sh" URL="https://ftp.ncbi.nih.gov/snp/archive/${CONSUMER_DBSNP_BUILD:?}/VCF/GCF_000001405.40.gz" -PINNED=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import genotype_rules_indels as m; \ -print(m.DBSNP[0], m.DBSNP[2], m.DBSNP[3], m.DBSNP[4], sep='\t')") +# The pinned record: the POS, REF and ALT constants of og-summary's apol1 module. +PINNED=$(awk '/^pub mod apol1/ { on = 1 } on && /^ const (POS|REF|ALT):/ { v = $NF; gsub(/[";_]/, "", v); x[$2] = v } + END { printf "rs71785313\t%s\t%s\t%s\n", x["POS:"], x["REF:"], x["ALT:"] }' \ + "${REPO}/rust/crates/og-summary/src/collect/group_c/traits.rs") echo "pinned: ${PINNED}" # same_record PINNED LIVE: the pinned rsid, POS and REF equal dbSNP's, and the pinned ALT is one of dbSNP's. @@ -73,12 +76,12 @@ PY done read_result() { - python3 - "$REPO" "${D}/$1" "$1" <<'PY' -import os, subprocess, sys -repo, d, s = sys.argv[1:] -sys.path.insert(0, os.path.join(repo, "bin")) -import collect_summary as cs -r = next(x for x in cs.genotype_rules(d, s) if x["rule"] == "APOL1") + "${REPO}/bin/og" summary collect --sample "$1" --sample-dir "${D}/$1" --assets "${REPO}/assets" \ + --out "${CASE_TMP}/$1.summary.json" > /dev/null + python3 - "${CASE_TMP}/$1.summary.json" <<'PY' +import json, sys +rules = json.load(open(sys.argv[1]))["sections"]["traits"]["data"]["rules"] +r = next(x for x in rules if x["rule"] == "APOL1") print(r["result"], r["g2"]["result"], r["g2"]["source"], r["g2"]["status"], sep="\t") PY } diff --git a/tests/e2e/genotype-rules-2-lpa-null.sh b/tests/e2e/genotype-rules-2-lpa-null.sh index f92c2bc11..891a1c72b 100755 --- a/tests/e2e/genotype-rules-2-lpa-null.sh +++ b/tests/e2e/genotype-rules-2-lpa-null.sh @@ -39,14 +39,20 @@ sample() { rm -rf "${G:?}/$1"; mkdir -p "${G}/$1/aligned" cp "$2" "${G}/$1/aligned/$1_sorted.bam"; cp "$2.bai" "${G}/$1/aligned/$1_sorted.bam.bai" } -SITES=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -print(','.join(map(str, t.MARKERS['LPA_KIV2_4925']['copies'])))") +# marker KEY copies|dbsnp: the copies (comma-separated) or the dbSNP record +# (rsID, POS, REF, ALT) of a copy-aware marker, from og-cn's table of them. +marker() { + awk -v k="$1" -v what="$2" '$0 ~ "key: \"" k "\"" { on = 1 } on && /^};/ { exit } on { b = b $0 "\n" } END { + if (what == "copies") { c = b; sub(/.*copies: &\[/, "", c); sub(/\].*/, "", c); gsub(/[ \n_]/, "", c); sub(/,$/, "", c); print c } + else { d = b; sub(/.*dbsnp: \(/, "", d); sub(/\).*/, "", d); gsub(/[ "_]/, "", d); split(d, x, ","); print x[1] "\t" x[3] "\t" x[4] "\t" x[5] } }' \ + "${REPO}/rust/crates/og-cn/src/markers.rs" +} +SITES=$(marker LPA_KIV2_4925 copies) check_eq "seven copies of the 4925G>A site" "$(tr ',' '\n' <<< "$SITES" | wc -l | tr -d ' ')" 7 # --- dbSNP orientation, online -------------------------------------------------------- URL="https://ftp.ncbi.nih.gov/snp/archive/${CONSUMER_DBSNP_BUILD:?}/VCF/GCF_000001405.40.gz" -PINNED=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -d = t.MARKERS['LPA_KIV2_4925']['dbsnp']; print(d[0], d[2], d[3], d[4], sep='\t')") +PINNED=$(marker LPA_KIV2_4925 dbsnp) same_record() { awk -F'\t' 'NR == 1 { id = $1; pos = $2; ref = $3; alt = $4; next } $1 == id && $2 == pos && $3 == ref && index("," $4 ",", "," alt ",") { ok = 1 } diff --git a/tests/e2e/genotype-rules-2-smn1-marker.sh b/tests/e2e/genotype-rules-2-smn1-marker.sh index 38cc4b746..e0e9134b5 100755 --- a/tests/e2e/genotype-rules-2-smn1-marker.sh +++ b/tests/e2e/genotype-rules-2-smn1-marker.sh @@ -30,14 +30,20 @@ W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } col() { awk -F'\t' -v k="$2" '$1 == k {print $2}' "$1" 2>/dev/null; } -SITES=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -print(','.join(map(str, t.MARKERS['SMN1_27134']['copies'])))") +# marker KEY copies|dbsnp: the copies (comma-separated) or the dbSNP record +# (rsID, POS, REF, ALT) of a copy-aware marker, from og-cn's table of them. +marker() { + awk -v k="$1" -v what="$2" '$0 ~ "key: \"" k "\"" { on = 1 } on && /^};/ { exit } on { b = b $0 "\n" } END { + if (what == "copies") { c = b; sub(/.*copies: &\[/, "", c); sub(/\].*/, "", c); gsub(/[ \n_]/, "", c); sub(/,$/, "", c); print c } + else { d = b; sub(/.*dbsnp: \(/, "", d); sub(/\).*/, "", d); gsub(/[ "_]/, "", d); split(d, x, ","); print x[1] "\t" x[3] "\t" x[4] "\t" x[5] } }' \ + "${REPO}/rust/crates/og-cn/src/markers.rs" +} +SITES=$(marker SMN1_27134 copies) check_eq "two copies of the marker site (SMN1, SMN2)" "$SITES" "70952074,70076654" # --- dbSNP orientation, online -------------------------------------------------------- URL="https://ftp.ncbi.nih.gov/snp/archive/${CONSUMER_DBSNP_BUILD:?}/VCF/GCF_000001405.40.gz" -PINNED=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -d = t.MARKERS['SMN1_27134']['dbsnp']; print(d[0], d[2], d[3], d[4], sep='\t')") +PINNED=$(marker SMN1_27134 dbsnp) same_record() { awk -F'\t' 'NR == 1 { id = $1; pos = $2; ref = $3; alt = $4; next } $1 == id && $2 == pos && $3 == ref && index("," $4 ",", "," alt ",") { ok = 1 } diff --git a/tests/e2e/str-extra-repeats-1-readings.sh b/tests/e2e/str-extra-repeats-1-readings.sh index 14c9806da..65981889f 100644 --- a/tests/e2e/str-extra-repeats-1-readings.sh +++ b/tests/e2e/str-extra-repeats-1-readings.sh @@ -120,12 +120,12 @@ unset REF_FASTA python3 - "$G" "$REPO" <<'PY' import os, sys g, repo = sys.argv[1], sys.argv[2] -sys.path.insert(0, os.path.join(repo, "bin")) -import traits_report +# The columns of step 42's traits table (og traits table). +OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids"] os.makedirs(f"{g}/SXA/traits", exist_ok=True) os.makedirs(f"{g}/SXA/cpic", exist_ok=True) with open(f"{g}/SXA/traits/SXA_traits.tsv", "w") as f: - f.write("\t".join(traits_report.OUT_COLUMNS) + "\n") + f.write("\t".join(OUT_COLUMNS) + "\n") f.write("rs78655421\tCFTR\tchr7\t117530975\tCFTR R117H\tG/G\tcalled\tNo R117H copy.\tno\t-\t32404922\n") with open(f"{g}/SXA/cpic/SXA_phenotypes.tsv", "w") as f: f.write("Gene\tDiplotype\tPhenotype\tStatus\nUGT1A1\t*1/*28\tIntermediate Metabolizer\tnon-normal\n") diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index 268a9ce99..721483261 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -5,8 +5,6 @@ # filenamesimporterslane bin/family_report.py COLUMNS,MISSING,SEX,NAME,BINS,die,missing,read_pedigree,build_members,bad_name,kinship_table,declared_relationship,degree,read_par,in_par bin/couple_report.py,bin/pedigree_svg.py,tests/test_family_report.py R4 bin/genome_lookup.py GENE_KEYS,RSID_RE,norm_chrom,parse_query,names_of,walk_mentions,flat bin/results_index.py,tests/test_results_index.py R5c -bin/pgx_outside_calls.py HLA_GENES,NO_CALL,PROBABLE,NOT_CALLABLE,LOW_GQ,is_call,same_diplotype,probable_cyp2d6 bin/collect_summary.py,tests/test_pgx_outside_calls.py R3 -bin/paraphase_report.py SMN1_EXON7,parascopy_smn1,compare_smn1 bin/collect_summary.py,bin/couple_report.py,tests/test_paralog_loci.py,tests/test_paraphase_report.py R4 +bin/paraphase_report.py SMN1_EXON7,parascopy_smn1,compare_smn1 bin/couple_report.py,tests/test_paralog_loci.py,tests/test_paraphase_report.py R4 bin/cnvpytor_to_vcf.py DEPTH_MIN_SIZE,delly_depth_contradicts bin/couple_report.py R4 -bin/annotsv_postfilter.py CONSENSUS_BP,KEEP,ADDED,open_text,chrom_key,sv_type,info_value,sample_format,sample_columns,read_consensus,consensus_support bin/collect_summary.py,tests/demo/make_demo_sample.py R3 -bin/traits_report.py COLUMNS,OUT_COLUMNS,RULE_OUT_COLUMNS,TRAITS_TABLE_RULES,TableError,read_table,MARKERS,MARKER_STATUSES bin/collect_summary.py,tests/demo/make_demo_sample.py,tests/test_traits_report.py,tests/test_traits_table.py,tests/test_collect_summary.py,tests/test_lpa_kiv2.py R3 +bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,CALL_FLOOR,COMMON_AF,RUN_ALL_STEPS,SEVERITY_KINDS,Unreadable,open_text,vcf_records,info_map,read_tsv,clinvar_variants,clinvar_combined_note,low_confidence,half_call,clinvar_zygosity,clinvar_row,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank,allele_key,parse_alleles bin/couple_report.py,bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_acmg_sf_rules.py,tests/test_ancestry_fine_report.py,tests/test_archaic_report.py,tests/test_callability.py,tests/test_chip_report.py,tests/test_collect_summary.py,tests/test_copy_number.py,tests/test_cross_engine_biallelic.py,tests/test_exomiser_report.py,tests/test_haplogroup_explore.py,tests/test_hla_report.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_locityper_report.py,tests/test_lpa_kiv2.py,tests/test_mei_report.py,tests/test_mito_extended.py,tests/test_mocha_report.py,tests/test_paralog_loci.py,tests/test_render_report.py,tests/test_results_index.py,tests/test_run_state.py,tests/test_traits_report.py,tests/test_viewer.py,tests/test_viral_report.py,tests/test_vntyper_report.py,tests/test_ystr_report.py R6b diff --git a/tests/test_ancestry_fine_report.py b/tests/test_ancestry_fine_report.py index 5e0831e28..1f00da38c 100755 --- a/tests/test_ancestry_fine_report.py +++ b/tests/test_ancestry_fine_report.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -"""og ancestry-fine, step 26b's commands, on made-up inputs, and what is left -of bin/ancestry_fine_report.py: +"""og ancestry-fine, step 26b's commands, on made-up inputs: panel-map a group of fewer than 10 unrelated people (one Oceanian, three San) is no reference and no painting reference, and the @@ -16,10 +15,7 @@ report fractions with the bootstrap range, merged painting rows, the painted shares, a PNG of the right size, the left-out group and the chromosome RFMix failed on, in the table, the text and - both pipeline reports (collect_summary.py, render_report.py) - the module the group names, colours and MIN_REFERENCES collect_summary.py - reads from it are the ones og writes, and its painting_png - draws the same pixels as og's S_painting.png + both pipeline reports (og summary collect, render_report.py) The Rust unit tests (rust/crates/og-ancestry) and tests/parity check each command in detail; this test runs them as the pipeline does, through bin/og. @@ -35,7 +31,6 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import ancestry_fine_report as af # noqa: E402 OG = os.environ.get("OG", "") FAILS = [] @@ -58,48 +53,6 @@ def write(path, text): f.write(text) -def png_pixels(data): - """The RGB rows of an 8-bit RGB PNG, every filter undone.""" - w, h = struct.unpack(">II", data[16:24]) - idat, pos = b"", 8 - while pos < len(data): - n, kind = struct.unpack(">I4s", data[pos:pos + 8]) - if kind == b"IDAT": - idat += data[pos + 8:pos + 8 + n] - pos += 12 + n - raw, stride, rows, prev = zlib.decompress(idat), 3 * w, [], bytearray(3 * w) - for y in range(h): - f, line = raw[y * (stride + 1)], bytearray(raw[y * (stride + 1) + 1:(y + 1) * (stride + 1)]) - for i in range(stride): - a = line[i - 3] if i >= 3 else 0 - b, c = prev[i], prev[i - 3] if i >= 3 else 0 - if f == 1: - line[i] = (line[i] + a) & 255 - elif f == 2: - line[i] = (line[i] + b) & 255 - elif f == 3: - line[i] = (line[i] + (a + b) // 2) & 255 - elif f == 4: - p = a + b - c - pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) - line[i] = (line[i] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)) & 255 - rows.append(bytes(line)) - prev = line - return (w, h, rows) - - -def first_difference(a, b): - """None if two PNGs hold the same pixels, else where they first differ.""" - (wa, ha, ra), (wb, hb, rb) = png_pixels(a), png_pixels(b) - if (wa, ha) != (wb, hb): - return f"size {wa}x{ha} vs {wb}x{hb}" - for y, (la, lb) in enumerate(zip(ra, rb)): - for x in range(wa): - if la[3 * x:3 * x + 3] != lb[3 * x:3 * x + 3]: - return f"pixel ({x}, {y}): {tuple(la[3 * x:3 * x + 3])} vs {tuple(lb[3 * x:3 * x + 3])}" - return None - - def run_all(tmp): # --- panel-map and check-map -------------------------------------------------- # 10 unrelated Yoruba, 3 unrelated San, one unrelated Papuan and two @@ -114,7 +67,7 @@ def run_all(tmp): "--paint-per-group", "50", "--out", samples) expect("panel-map: exits 0", (r.returncode, r.stderr), (0, "")) expect("panel-map: the install log names the groups left out", - f"Left out of the model, fewer than {af.MIN_REFERENCES} unrelated reference people: " + "Left out of the model, fewer than 10 unrelated reference people: " "Oceania (continental and regional group), Southern Africa (regional group)" in r.stdout, True) rows = [line.split("\t") for line in open(samples).read().splitlines()] by = {row[0]: row for row in rows[1:]} @@ -244,20 +197,7 @@ def run_all(tmp): expect("report: the text says chr19 was not painted, and chr20 still was", ("Not painted: RFMix failed on chr19" in text, "of chr20:" in text), (True, True)) - # --- the module and og agree on the names collect_summary.py reads --------------- - expect("module: each group name in og's table is the module's label", - [row[2] for row in table[1:] if row[0] in ("continent", "region", "painting")], - [af.label("region" if row[0] == "region" else "continent", row[1]) - for row in table[1:] if row[0] in ("continent", "region", "painting")]) - expect("module: MIN_REFERENCES is 10, as og's", af.MIN_REFERENCES, 10) - expect("module: a colour for each continental group", sorted(af.COLOURS), sorted(af.LABELS["continent"])) - # The legend collect_summary.py draws from af.COLOURS, and the docs demo's - # painting_png, must match og's picture: the same rows give the same - # pixels (the compressed bytes may differ). - demo = af.painting_png([(c, int(s), int(e), int(hp), code) for c, s, e, hp, code in bed]) - expect("module: painting_png draws og's pixels, colour for colour", first_difference(demo, png), None) - - # Both pipeline reports (collect_summary.py, then render_report.py) carry them. + # Both pipeline reports (og summary collect, then render_report.py) carry them. import collect_summary # noqa: E402 import render_report # noqa: E402 sd = os.path.join(tmp, "S") diff --git a/tests/test_e2e_scope.sh b/tests/test_e2e_scope.sh index b7717c684..9ea0189b6 100755 --- a/tests/test_e2e_scope.sh +++ b/tests/test_e2e_scope.sh @@ -34,7 +34,7 @@ paths() { printf '# a comment\n\n' printf '%s\n' "01-base${TAB}-" "02-base${TAB}-" "lpa-1${TAB}modules/local/lpa/* scripts/46-lpa.sh" \ "nextflow-from-fastq-1-bash${TAB}scripts/01b-fastp-qc.sh" "nextflow-from-fastq-2-nextflow${TAB}-" \ - "traits-1${TAB}-" "traits-2${TAB}bin/traits_report.py" + "traits-1${TAB}-" "traits-2${TAB}bin/og" } paths > "$E2E_PATHS" @@ -53,14 +53,14 @@ broken "a second row for a case" "traits-2 has more than one row" "^NONE" "trait broken "a row for a case that is not a file" "row for gone" "^NONE" "gone${TAB}main.nf" broken "a base row with globs" "base case 01-base" "^01-base${TAB}" "01-base${TAB}main.nf" broken "a glob that matches no file" "bin/no_such_helper.py matches no tracked file" "^traits-2${TAB}" \ - "traits-2${TAB}bin/traits_report.py bin/no_such_helper.py" + "traits-2${TAB}bin/og bin/no_such_helper.py" broken "a module glob that matches no file" "modules/local/nope/\\* matches no tracked file" "^lpa-1${TAB}" \ "lpa-1${TAB}modules/local/nope/*" broken "a row of one column" "has 1 columns" "^traits-2${TAB}" "traits-2" broken "a glob that matches a file outside the e2e scope" "docs/testing.md matches docs/testing.md, which is outside the e2e scope" \ - "^traits-2${TAB}" "traits-2${TAB}bin/traits_report.py docs/testing.md" + "^traits-2${TAB}" "traits-2${TAB}bin/og docs/testing.md" broken "a wildcard glob that reaches outside the e2e scope" "docs/\\* matches docs/" "^traits-2${TAB}" \ - "traits-2${TAB}bin/traits_report.py docs/*" + "traits-2${TAB}bin/og docs/*" mkdir -p "${WORK}/nosmoke" cp "${E2E_CASE_DIR}"/*.sh "${WORK}/nosmoke/" rm "${WORK}/nosmoke/nextflow-from-fastq-2-nextflow.sh" @@ -86,7 +86,7 @@ expect "a change outside the scope runs nothing" "" tests/test_cpic_parser.py .g expect "main.nf runs every shard case" "lpa-1 ${SMOKE} traits-1 traits-2" main.nf docs/testing.md expect "a workflow file runs every shard case" "lpa-1 ${SMOKE} traits-1 traits-2" workflows/pgx.nf expect "the e2e runner runs every shard case" "lpa-1 ${SMOKE} traits-1 traits-2" tests/e2e/lib.sh -expect "one helper runs the smoke cases, its case and the case that needs it" "${SMOKE} traits-1 traits-2" bin/traits_report.py +expect "one helper runs the smoke cases, its case and the case that needs it" "${SMOKE} traits-1 traits-2" bin/og expect "a file in one module runs that module's case" "lpa-1 ${SMOKE}" modules/local/lpa/main.nf expect "an in-scope path no case runs runs the smoke cases" "${SMOKE}" scripts/ci/check-name.sh expect "a committed fixture a case reads is in the scope" "${SMOKE}" tests/fixtures/consumer_export/sites.tsv.gz @@ -96,11 +96,11 @@ expect "a base step runs the smoke cases only (the base leg runs every base case printf '%s\n' traits-2 > "${WORK}/rows.txt" ROWS="${WORK}/rows.txt" expect "a changed table row runs its case and what it needs" "${SMOKE} traits-1 traits-2" docs/x.md expect "a deleted case file runs the smoke cases" "${SMOKE}" tests/e2e/gone-1.sh -if grep -q 'traits-1: needed by traits-2' "${WORK}/why.txt" 2> /dev/null || { pick bin/traits_report.py > /dev/null; grep -q 'traits-2: runs bin/traits_report.py' "${WORK}/why.txt"; }; then +if grep -q 'traits-1: needed by traits-2' "${WORK}/why.txt" 2> /dev/null || { pick bin/og > /dev/null; grep -q 'traits-2: runs bin/og' "${WORK}/why.txt"; }; then ok "select says why it picked each case" else bad "select gave no reasons: $(cat "${WORK}/why.txt")"; fi paths | grep -v "^traits-1${TAB}" > "${WORK}/norow.tsv" -printf '%s\n' bin/traits_report.py > "${WORK}/changed.txt" +printf '%s\n' bin/og > "${WORK}/changed.txt" if E2E_PATHS="${WORK}/norow.tsv" "$S" select "${WORK}/changed.txt" > /dev/null 2>&1; then bad "select ran with a case that has no row" else ok "select refuses a table with a case that has no row"; fi @@ -111,7 +111,7 @@ diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -3 +3,2 @@ -# an old comment -+traits-2${TAB}bin/traits_report.py ++traits-2${TAB}bin/og +lpa-1${TAB}- diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv --- a/scripts/ci/e2e-case-times.tsv @@ -131,7 +131,7 @@ have=$(awk '/^ push:/ {p = 1; next} p && /^ [a-z_]+:/ {exit} p && /^ - / {sub(/^ - /, ""); gsub(/\047/, ""); print}' "${REPO}/.github/workflows/e2e.yml" | sort) if [ "$("$S" scope)" = "$have" ]; then ok "the push paths of e2e.yml are the scope" else bad "push paths and scope differ: $(diff <("$S" scope) <(echo "$have") || true)"; fi -printf '%s\n' bin/traits_report.py > "${WORK}/changed.txt" +printf '%s\n' bin/og > "${WORK}/changed.txt" "$S" select "${WORK}/changed.txt" > "${WORK}/real.txt" 2> /dev/null if [ -s "${WORK}/real.txt" ] && E2E_SELECT="${WORK}/real.txt" "${REPO}/scripts/ci/e2e-shards.sh" plan 2 > /dev/null; then ok "a real pick splits: $(paste -sd' ' "${WORK}/real.txt")" diff --git a/tests/test_pgx_outside_calls.py b/tests/test_pgx_outside_calls.py index 9d4fe22a7..33664a9b1 100644 --- a/tests/test_pgx_outside_calls.py +++ b/tests/test_pgx_outside_calls.py @@ -31,11 +31,9 @@ 11. (the star alleles of a diplotype and the defining-variant evidence of one call are og's own unit tests: cargo test -p og-steps pypgx_summary) 13. pypgx alone with the depth check ok and PASS defining variants: the - reports' probable call (bin/pgx_outside_calls.py, kept for the - summary), while the outside-call file stays empty and step 36's row - stays indeterminate; no probable call with a failed depth check, a - usable Cyrius call, a low-GQ or non-PASS defining variant, or a - summary without the columns + outside-call file stays empty and step 36's row stays indeterminate + (the reports' probable call itself is og summary collect's rule, + tested in rust/crates/og-summary/src/collect/pgx.rs) 14. the whole subcommand on pypgx results archives, pypgx's own tables (rows of pypgx 0.26.0's gene and allele tables, tests/fixtures/pgx/ summary/) and an indexed VCF: GSTT1 not callable on this reference @@ -228,8 +226,8 @@ def medications(text): report_has("10 with the table missing, the report says to run step 36 again", gone, "Run step 36 again, then steps 07 and 27.") # --- step 32's summary and the probable CYP2D6 call ------------------------------- -sys.path.insert(0, os.path.join(REPO, "bin")) -import pgx_outside_calls as poc # noqa: E402 +# The rule for pypgx's call alone (probable_cyp2d6) is og summary collect's, +# tested in rust/crates/og-summary/src/collect/pgx.rs. SUMMARY_HEADER = ["Gene", "Diplotype", "Phenotype", "CNV_call", "Source", "Status", "Defining_variants", "Min_GQ", "Weakest_variant", "Defining_filter", "Quality"] @@ -244,26 +242,6 @@ def same(desc, got, want): # 13. -ROW = {"Gene": "CYP2D6", "Diplotype": "*1/*2", "Phenotype": "Normal Metabolizer", "Defining_filter": "PASS", - "Quality": "ok"} -same("13 pypgx alone, depth ok, PASS defining variants: probable", - poc.probable_cyp2d6(ROW, None, "ok"), - {"diplotype": "*1/*2", "phenotype": "Normal Metabolizer", "label": "probable (one caller: pypgx)"}) -same("13 Cyrius ran and made no call: still one caller, probable", - (poc.probable_cyp2d6(ROW, ("None", "Not_assigned_to_haplotypes"), "ok") or {}).get("label"), poc.PROBABLE) -same("13 red control: a failed depth check gives no probable call", poc.probable_cyp2d6(ROW, None, "unreliable"), None) -same("13 a usable Cyrius call: the two callers decide, no probable call", - poc.probable_cyp2d6(ROW, ("*1/*2", "PASS"), "ok"), None) -same("13 a low-GQ defining variant: no probable call", - poc.probable_cyp2d6(dict(ROW, Quality=poc.LOW_GQ), None, "ok"), None) -same("13 a defining variant not PASS: no probable call", - poc.probable_cyp2d6(dict(ROW, Defining_filter="not PASS: chr22:42128945 C>T RefCall"), None, "ok"), None) -same("13 an allele pypgx's table lacks: no probable call", - poc.probable_cyp2d6(dict(ROW, Defining_filter="not in pypgx's table: *99"), None, "ok"), None) -same("13 a defining variant with no GQ: no probable call", - poc.probable_cyp2d6(dict(ROW, Quality="GQ not in the VCF: check the reads"), None, "ok"), None) -same("13 a summary without the columns: no probable call", - poc.probable_cyp2d6({"Diplotype": "*1/*2", "Phenotype": "NM"}, None, "ok"), None) with tempfile.TemporaryDirectory() as tmp: summ = os.path.join(tmp, "pypgx.tsv") with open(summ, "w") as f: From d2bb659b9fca14b3af643de622843cd2938a94d5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:37:43 +0100 Subject: [PATCH 23/86] docs: name og summary collect where the docs named the Python collector Bead: pgp-bs9u.45 --- docs/05-annotsv.md | 2 +- docs/08-hla-typing.md | 2 +- docs/09-str-expansions.md | 2 +- docs/11-roh-analysis.md | 4 ++-- docs/17-cpsr.md | 2 +- docs/23-clinical-filter.md | 2 +- docs/24-html-report.md | 2 +- docs/33-sample-qc.md | 2 +- docs/36-pgx-consensus.md | 2 +- docs/42-traits.md | 4 ++-- docs/interpreting-results.md | 2 +- docs/nextflow.md | 2 +- docs/run-state.md | 2 +- docs/rust.md | 7 +++---- rust/README.md | 2 +- tests/demo/make_demo_sample.py | 2 +- 16 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docs/05-annotsv.md b/docs/05-annotsv.md index 2d4798eb0..eb61026c3 100644 --- a/docs/05-annotsv.md +++ b/docs/05-annotsv.md @@ -126,7 +126,7 @@ Some class 4 and 5 calls are common or unreliable for a reason AnnotSV does not The regions are in `assets/sv_rule_regions.tsv`, one row each (`chrom`, `start` 0-based, `end`, `rule`, `action`, `name`, `source`): 2,409 coding tandem repeats, the six IG and TR loci, the MHC and 10 pseudogenes. `scripts/ci/build-sv-rule-regions.sh` writes it from four pinned downloads, each checked against its sha256: TRExplorer v2.1's GRCh38 tandem repeat catalog (the file upstream rebuilt in place on 2026-10-30, sha256 `df9c23c4…`; every repeat of 50 bp or more on a MANE Select coding exon of GENCODE 50), GENCODE 50's IG and TR genes, the MHC row of the GRCh38 assembly regions report, and HGNC's complete set of 2026-10-06 (each pseudogene whose alias or previous symbol is the approved symbol of a gene with a GenCC curation). The file is about 250 KB and ships with the repository, so the step downloads nothing. The e2e case `followups-ci-3-sv-rule-regions` rebuilds it with `scripts/ci/build-sv-rule-regions.sh --check`, which fails when one byte differs, so a hand edit or a drifted pin cannot ship. Step 05 mounts it into the AnnotSV image and the Nextflow `ANNOTSV` gets it as an input. -How many callers saw each SV comes from step 22's consensus (`sv_merged/_sv_consensus.vcf.gz`), which runs after this step, and beside `ANNOTSV` in Nextflow. So the reports (`bin/collect_summary.py`) join it when they are written: an SV matches the consensus record of the same type whose two breakpoints lie within 1,000 bp of its own, step 22's distance, and its support is that record's `SUPP` with the callers of `SUPP_VEC`. An SV in no consensus record was made by one caller and shows as "single caller". Without the consensus file the support is "not known", never 1. `og sv postfilter --consensus` adds the same `Support` and `Support_callers` columns to the table. +How many callers saw each SV comes from step 22's consensus (`sv_merged/_sv_consensus.vcf.gz`), which runs after this step, and beside `ANNOTSV` in Nextflow. So the reports (`og summary collect`) join it when they are written: an SV matches the consensus record of the same type whose two breakpoints lie within 1,000 bp of its own, step 22's distance, and its support is that record's `SUPP` with the callers of `SUPP_VEC`. An SV in no consensus record was made by one caller and shows as "single caller". Without the consensus file the support is "not known", never 1. `og sv postfilter --consensus` adds the same `Support` and `Support_callers` columns to the table. Both reports list every class 4 and 5 SV in a section of their own, with its support, the exon-edge flag and the frequency. The bash report (`scripts/24-html-report.sh`, which `run-all.sh` runs) reads the table from the sample folder. The Nextflow `HTML_REPORT` gets the table from `ANNOTSV` and, with `survivor_merge` in `--tools`, the consensus, so a report written by `nextflow run` alone has the same section. A report of a run without AnnotSV lists it under the steps not run. diff --git a/docs/08-hla-typing.md b/docs/08-hla-typing.md index 3c766f372..5926f8849 100644 --- a/docs/08-hla-typing.md +++ b/docs/08-hla-typing.md @@ -120,7 +120,7 @@ docker run --rm \ ## What the Report Derives -The report (step 24 and the text report) lists each T1K row and then derives three things from it. The code is `sec_hla` and the helpers below it in `bin/collect_summary.py`; `tests/test_hla_report.py` checks every rule on synthetic T1K rows. The genotypes are unphased: T1K gives two alleles per gene, not which chromosome each sits on. +The report (step 24 and the text report) lists each T1K row and then derives three things from it. The code is the HLA section of `og summary collect` (`rust/crates/og-summary/src/collect/group_c/hla.rs`); `tests/test_hla_report.py` checks every rule on synthetic T1K rows. The genotypes are unphased: T1K gives two alleles per gene, not which chromosome each sits on. **Coeliac DQ.** From the DQA1 and DQB1 rows: diff --git a/docs/09-str-expansions.md b/docs/09-str-expansions.md index d0bcd4f2c..c46b8a99e 100644 --- a/docs/09-str-expansions.md +++ b/docs/09-str-expansions.md @@ -47,7 +47,7 @@ Two short repeats that change how other findings read are not disease expansions Each entry has a `NoThresholds` reason and no Stranger thresholds: step 9b gives them no status, and the step 09 log prints them as "Not graded" with that reason. The counts are in the VCF (`REPCN`) and the JSON, one record per repeat: the report names the two CFTR records by their `VARID`, `CFTR_TG` and `CFTR_T`. -The reports read the counts (`bin/collect_summary.py`, under the repeat expansions section, `readings` in `summary.json`): +The reports read the counts (`og summary collect`, under the repeat expansions section, `readings` in `summary.json`): - **CFTR**: the TG and poly-T counts of both copies, such as `TG11-7T/TG11-7T`. ExpansionHunter writes the two tracts as two records, so when both the TG and the poly-T counts differ between the copies, which TG sits with which poly-T is not known: the line says `TG11/TG12 5T/7T` and names both pairings. A 5T copy gets the 5T line: 5T is a variable-penetrance allele, more often disease-causing with TG12 or TG13 on the same copy, and it matters when the other copy carries a CF-causing variant. The line then reads R117H (rs78655421), a row of step 42's traits table: no R117H copy; R117H and 5T in cis when one of them is on both copies; otherwise phase unknown, because R117H lies 17.6 kb from the tract, farther than a read pair spans. No 5T copy gives no 5T line, but the poly-T still matters beside R117H (milder on a 7T or 9T copy, CF-causing with 5T in cis), so the line still reads R117H and says which poly-T copy it sits on. A traits table from an earlier run is read and said to be so; one of a failed or unreadable step 42 is not read. - **UGT1A1**: the TA count of each copy as its allele (`*36`, `*1`, `*28`, `*37`), and the number of decreased-function copies (`*28`, `*37`) compared with PharmCAT's UGT1A1 call of step 27, which reads `*28` from the small-variant calls with the reference-block fill of `og pgx refblock-fill`. The line says whether the two agree; when they differ, read the reads before using either. A TA length outside TA5 to TA8 has no named allele, so the line makes no function claim and does not compare it. A step 27 table from an earlier run is compared and said to be so; one of a failed or unreadable step is not compared. PharmCAT's call is still the one step 27's drug guidance uses. diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 0435e38b2..20a8146bb 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -42,7 +42,7 @@ For chip data (no `FORMAT/PL`), the script adds `-G30`. ### Centromere and heterochromatin bands -Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `bin/collect_summary.py` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. +Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `og summary collect` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. The reports count autosomes only: Segments, Total and Largest keep chr1 to chr22, so they leave out chrX and chrY (a man's single X reads as one long run) and any other contig (chrM, chrEBV, chrUn_*), and the part of each segment in a band. They say how many chrX and chrY segments and how many megabases in bands were left out. @@ -89,7 +89,7 @@ A flagged chromosome is looked up in `assets/imprinted_regions.tsv`, the chromos | 15q11-q13, chr15:20,500,000-33,400,000 (SNRPN, UBE3A) | Prader-Willi syndrome | Angelman syndrome | 22237428, 27615419 | | chr20 (20q13.32, GNAS) | Mulchandani-Bhoj-Conlin syndrome (growth failure) | Pseudohypoparathyroidism type 1B | 26248010, 29959430 | -`bin/collect_summary.py` holds the same rows (a unit test checks that the two agree), because the Nextflow report task reads `bin/` but not `assets/`. A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. +`og summary collect` holds the same rows (`IMPRINTED_REGIONS` in `rust/crates/og-summary`; a unit test checks that the two agree). A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. What this cannot tell, and the reports say so: diff --git a/docs/17-cpsr.md b/docs/17-cpsr.md index e542e1feb..abf28e100 100644 --- a/docs/17-cpsr.md +++ b/docs/17-cpsr.md @@ -120,7 +120,7 @@ Run by hand like this, `--sample_id` must be 3 to 40 characters long. The script - `${SAMPLE}.cpsr.grch38.html` — Interactive HTML report with classified variants - `${SAMPLE}.cpsr.grch38.classification.tsv.gz` — Tab-separated variant classifications (gzipped; read it with `zcat`) - Every variant in the panel genes gets one of the five ACMG/AMP classes: Pathogenic, Likely pathogenic, VUS, Likely benign, Benign -- In the TSV, `CLASSIFICATION` is the final class and `ASSERTION_AUTHORITY` says whose it is: ClinVar's, unless ClinVar has no record or a conflicted one, then CPSR's own (`CPSR_CLASSIFICATION`, which every variant has). PCGR 2.2.5 named the final column `FINAL_CLASSIFICATION`; the reports (`bin/collect_summary.py`) read either. The file names are the same in both versions. +- In the TSV, `CLASSIFICATION` is the final class and `ASSERTION_AUTHORITY` says whose it is: ClinVar's, unless ClinVar has no record or a conflicted one, then CPSR's own (`CPSR_CLASSIFICATION`, which every variant has). PCGR 2.2.5 named the final column `FINAL_CLASSIFICATION`; the reports (`og summary collect`) read either. The file names are the same in both versions. - `${SAMPLE}_cpsr_cdna_check.tsv` — step 6's cDNA read check on CPSR's P/LP rows (final class or CPSR's own), in the columns of step 6's `clinvar/${SAMPLE}_clinvar_cdna_check.tsv`. The rows are read from `VAR_ID` (or `GENOMIC_CHANGE`) and keep CPSR's contig names (`13`, not `chr13`); the BAM is read under either name. It needs the sample's BAM, `aligned/${SAMPLE}_sorted.bam` with its `.bai` (`ALIGN_DIR` picks another folder), and the GENCODE exons `setup.sh` installs; without either, every row is "not checked" with the reason. In Nextflow step 6's `CLINVAR_CDNA_CHECK` process runs it after `CPSR` (source `cpsr`) under the name `CPSR_CDNA_CHECK`, so the run's process table and the live view count it under step 17, with the samplesheet's BAM and `--gencode_exons`, and publishes the table to `cpsr/`; the HTML report puts it beside each P/LP row there too. Its image comes from a `withName: 'CPSR_CDNA_CHECK'` selector (a setting for step 6's `CLINVAR_CDNA_CHECK` does not reach it). ### What the reports show diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index 66f03986c..4f4eec752 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -86,7 +86,7 @@ The Nextflow `CLINICAL_FILTER` module applies the same tiers, ranking, frequency ### Secondary-findings genes (ACMG SF v3.3) -Both reports list, beside the clinical filter's counts, the secondary findings in the 84 genes of the ACMG SF v3.3 list (Lee et al., Genet Med 2025;27(8):101454; the genes in which the ACMG recommends reporting pathogenic variants found by chance, such as BRCA1, LDLR, MYBPC3 or TTN): the step 6 ClinVar hits in those genes, the clinical filter's records with a HIGH-impact consequence on the ranked transcript in them, and step 5's AnnotSV class 4 and 5 structural variants in the genes whose rule allows them. Each gene's rule from Table 1 of the list is a row of `assets/acmg_sf.tsv`: one variant in a dominant gene, two in a recessive one (MUTYH, BTD, ATP7B and others; a homozygote, or two variants of unknown phase), HFE only as homozygous p.C282Y, GLA and OTC also in a heterozygous female, ABCD1 hemizygous, homozygous or with two heterozygous variants. A variant the rule does not report is listed apart with the reason; a pseudodeficiency allele is not counted, and a BTD finding with the p.Asp444His hypomorph says partial, not profound. Of the reported HIGH-impact records the reports count as "rare HIGH impact" only the rare ones with a sound call; a weak call or an unknown frequency is listed with its mark and counted apart, and a common one is left out. The list version is `ACMG_SF_VERSION` in `bin/collect_summary.py`. It is a list to review with a clinician: a rare HIGH-impact variant is not a ClinVar classification. [Interpreting results](interpreting-results.md#carrier-status-is-not-disease) gives every rule. Step 17 (CPSR) reports the ACMG secondary findings with its own classification. +Both reports list, beside the clinical filter's counts, the secondary findings in the 84 genes of the ACMG SF v3.3 list (Lee et al., Genet Med 2025;27(8):101454; the genes in which the ACMG recommends reporting pathogenic variants found by chance, such as BRCA1, LDLR, MYBPC3 or TTN): the step 6 ClinVar hits in those genes, the clinical filter's records with a HIGH-impact consequence on the ranked transcript in them, and step 5's AnnotSV class 4 and 5 structural variants in the genes whose rule allows them. Each gene's rule from Table 1 of the list is a row of `assets/acmg_sf.tsv`: one variant in a dominant gene, two in a recessive one (MUTYH, BTD, ATP7B and others; a homozygote, or two variants of unknown phase), HFE only as homozygous p.C282Y, GLA and OTC also in a heterozygous female, ABCD1 hemizygous, homozygous or with two heterozygous variants. A variant the rule does not report is listed apart with the reason; a pseudodeficiency allele is not counted, and a BTD finding with the p.Asp444His hypomorph says partial, not profound. Of the reported HIGH-impact records the reports count as "rare HIGH impact" only the rare ones with a sound call; a weak call or an unknown frequency is listed with its mark and counted apart, and a common one is left out. The list version is `ACMG_SF_VERSION` in `og summary collect` (`rust/crates/og-summary`). It is a list to review with a clinician: a rare HIGH-impact variant is not a ClinVar classification. [Interpreting results](interpreting-results.md#carrier-status-is-not-disease) gives every rule. Step 17 (CPSR) reports the ACMG secondary findings with its own classification. ## Runtime diff --git a/docs/24-html-report.md b/docs/24-html-report.md index 889f675f2..d5551fecb 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -10,7 +10,7 @@ The pipeline produces output across many directories in different formats (VCF, ## Tool -`bin/collect_summary.py` reads every step's output once into `${SAMPLE}/summary.json`, and `bin/render_report.py` renders the report from it. `scripts/generate-report.sh` renders the text report from the same summary, so the two reports cannot disagree about a count. Both are Python standard library only. +`og summary collect` (`rust/crates/og-summary`, run by `bin/render_report.py`) reads every step's output once into `${SAMPLE}/summary.json`, and `bin/render_report.py` renders the report from it. `scripts/generate-report.sh` renders the text report from the same summary, so the two reports cannot disagree about a count. Both are Python standard library only. ## Docker Image diff --git a/docs/33-sample-qc.md b/docs/33-sample-qc.md index 94e085561..2aed8fd77 100644 --- a/docs/33-sample-qc.md +++ b/docs/33-sample-qc.md @@ -19,7 +19,7 @@ Contamination only warns. A contaminated sample is still your sample; its calls ## Docker Image - `SOMALIER_IMAGE` - `VERIFYBAMID2_IMAGE` -- `PYTHON_IMAGE` (the verdict, from `bin/collect_summary.py`, and the mosaic estimate, from `og qc sex-loss`, the Rust binary `scripts/setup.sh --og` builds) +- `PYTHON_IMAGE` (the verdict, from `og summary collect`, and the mosaic estimate, from `og qc sex-loss`, the Rust binary `scripts/setup.sh --og` builds) Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. diff --git a/docs/36-pgx-consensus.md b/docs/36-pgx-consensus.md index 64779cb30..59427ba6a 100644 --- a/docs/36-pgx-consensus.md +++ b/docs/36-pgx-consensus.md @@ -63,7 +63,7 @@ For CYP2D6 the `Reason` is one of: the depth check's message (multi-mapped reads ## Notes - Without Cyrius, CYP2D6 is always `indeterminate` here: that is the point, not a fault. Install Cyrius (`setup.sh --cyrius`, non-commercial licence) to get an agreed call; `run-all.sh` then runs step 21 in a default run. -- **A probable call, never an outside call.** When pypgx alone called CYP2D6 (Cyrius did not run, or made no usable call), the depth check passed, and every variant that defines its star alleles is in the VCF as a `PASS` call at or above the GQ floor (step 32's `Defining_filter` and `Quality` columns), both reports show pypgx's diplotype and phenotype labelled `probable (one caller: pypgx)`. This table still says `indeterminate` for it and nothing reaches PharmCAT, so no drug guidance rests on it. The rule is `probable_cyp2d6` in `bin/pgx_outside_calls.py`, which the reports import; a failed depth check, a usable Cyrius call (then the two callers decide), a defining variant that is missing, not `PASS`, below the GQ floor or without a GQ, a star allele pypgx's table lacks, or a summary from before these columns gives no probable call. +- **A probable call, never an outside call.** When pypgx alone called CYP2D6 (Cyrius did not run, or made no usable call), the depth check passed, and every variant that defines its star alleles is in the VCF as a `PASS` call at or above the GQ floor (step 32's `Defining_filter` and `Quality` columns), both reports show pypgx's diplotype and phenotype labelled `probable (one caller: pypgx)`. This table still says `indeterminate` for it and nothing reaches PharmCAT, so no drug guidance rests on it. The rule is the probable call of `og summary collect` (`rust/crates/og-summary/src/collect/pgx.rs`); a failed depth check, a usable Cyrius call (then the two callers decide), a defining variant that is missing, not `PASS`, below the GQ floor or without a GQ, a star allele pypgx's table lacks, or a summary from before these columns gives no probable call. - A Cyrius call counts only with Filter `PASS`; `None`, an ambiguous call (two diplotypes) or `CYP2D6_depth_unreliable` is no call. - The two diplotypes must be the same up to the order of the alleles. The one passed to PharmCAT is pypgx's. - PharmCAT turns the HLA alleles into its own phenotypes (`*57:01 positive`, `*58:01 negative` and so on) and gives the drug guidance for them; step 27 marks those genes `[outside call]`. diff --git a/docs/42-traits.md b/docs/42-traits.md index ad3aa25cf..b43df3ed9 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -108,12 +108,12 @@ The rules today: The six rules after TAS2R38 need no phase. Their sites lie close together on one copy (the two APOL1 G1 changes were in perfect linkage disequilibrium in Genovese et al. 2010; the 9p21 and 17q21.31 tags each mark one haplotype), or, for HFE, a double heterozygote reads as one change on each copy. A copy the rule does not describe, such as a 9p21 copy with only one of the two risk alleles, is `other_haplotype`, not read. What each rule leaves out, said in its note: -- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`bin/genotype_rules_indels.py`, through `bin/collect_summary.py`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. +- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. - SERPINA1: M is any copy with neither S nor Z; rarer deficiency and null alleles are not read. A serum level confirms a deficiency. - HFE: the C282Y homozygote is also a secondary finding (`assets/acmg_sf.tsv`, [step 23](23-clinical-filter.md)); the rule gives the reading of every genotype. - GC: the protein levels are those of Powe et al. (2013), whose monoclonal assay was questioned in later letters. A protein type, not a vitamin D deficiency. -Two more markers of the planned list are not rules here, because each sits in sequence the genome holds more than once and a gVCF genotype at one position cannot read it. They are read from the BAM by the step that counts the copies, as a fraction of copies rather than a genotype (`MARKERS` in `bin/traits_report.py`, which pins each one's dbSNP b157 record and the GRCh38 sequence around every copy): +Two more markers of the planned list are not rules here, because each sits in sequence the genome holds more than once and a gVCF genotype at one position cannot read it. They are read from the BAM by the step that counts the copies, as a fraction of copies rather than a genotype (`MARKERS` in `rust/crates/og-cn/src/markers.rs`, which pins each one's dbSNP b157 record and the GRCh38 sequence around every copy): - **LPA KIV-2 4925G>A** (Coassin et al. 2017, PMID 28444229), a splice variant that lowers the Lp(a) made from the chromosome that carries it. GRCh38 holds the site in each of its six KIV-2 units and in KIV-1. [Step 46](46-lpa.md) reads it beside the KIV-2 count, and the reports print it under the count. - **SMN1 g.27134T>G** (rs143838139; Luo et al. 2014, PMID 23788250), a marker of SMN1 duplication alleles that SMN2 holds too. [Step 35](35-paralogs.md) reads it beside the SMN1 copy number, and the reports print it under the SMN1 copies. It refines the chance of a silent 2+0 carrier for a two-copy result, and its line always says that it does not prove 2+0. diff --git a/docs/interpreting-results.md b/docs/interpreting-results.md index 9aa0a7bd7..576e963c2 100644 --- a/docs/interpreting-results.md +++ b/docs/interpreting-results.md @@ -76,7 +76,7 @@ The most common "pathogenic" finding in any genome is **heterozygous carrier sta - BTD with p.Asp444His: reported as partial, not profound, biotinidase deficiency; - AnnotSV class 4 and 5 structural variants (step 5) in BRCA1, BRCA2, MLH1, MSH2 and LDLR reach the same card, and so does a deletion of the 3' end of EPCAM (exons 8 and 9), which silences MSH2 (Kovacs et al., Hum Mutat 2009;30(2):197-203). -**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`bin/collect_summary.py`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: +**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`og summary collect`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: - **biallelic** when the reads show the two hits on different copies. Either the hit is called homozygous inside the deletion, so it sits on the one copy left (the line and the carrier list call it hemizygous), or both calls carry read-backed phase (a `|` genotype with the same phase set, PS) on different copies; - **possibly biallelic** otherwise: the phase is unknown. Most short-read calls carry no phase, so this is the usual line. Step 05's checked table keeps no genotype column yet, and step 35's counts have no phase, so in a pipeline run a deletion pairs as possibly biallelic unless the hit is called homozygous inside it. A parent's genome (step 38, [family analysis](multi-sample.md#family-analysis-step-38-opt-in)) or long reads can settle it; diff --git a/docs/nextflow.md b/docs/nextflow.md index 0f15b62c5..07224e7ed 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -359,7 +359,7 @@ Where a module and its script differ on purpose: | Mito variants (20) | extracts the chrM reads with `samtools view`; annotates the calls with `clinvar/clinvar_chr.vcf.gz` and the gnomAD chrM sites under `gnomad_mito/` when they are installed | with GATK `PrintReads`. Mutect2 applies its own read filters to either, so the calls are expected to match; CI does not compare them. Both mark possible NuMTs at the median autosomal depth from mosdepth (step 16b, or `MOSDEPTH` when `mosdepth` is in `--tools`). `MITO_VARIANTS_ANNOTATE` runs the same annotation (`og mito annotate`) with the files of `--mito_clinvar` and `--gnomad_mito`; without them the table names each file as not given and leaves its columns `.`. `run-all.sh` passes `--mito_clinvar` when `clinvar/clinvar_chr.vcf.gz` and its `.tbi` are installed, and `--gnomad_mito` when the gnomAD chrM sites are | | AnnotSV (05) | annotates step 15's duphold-filtered calls; without them, or when Manta's calls are newer, Manta's calls, and says so | `ANNOTSV` always reads `DUPHOLD_FILTER`'s output: `annotsv` in `--tools` needs `duphold` and `manta` | | Y haplogroup (37) | reads the sex step 16 (indexcov) infers | runs on the rows whose samplesheet sex is male, which `INDEXCOV` has checked against the reads | -| HTML report (24) | renders every section from `bin/collect_summary.py`'s summary | `HTML_REPORT` runs the same code on the outputs of this run's steps, among them the cDNA read check of the ClinVar hits, the CYP2D6 depth check, the phased compound-het pairs, the RFC1 motif count, AnnotSV's class 4 and 5 table, the chrM annotation and step 35's HBA1/HBA2 count, CYP21A2 and gene-dosage tables. It also gets the sample's gVCF (DEEPVARIANT's or the `gvcf` column's, linked as `vcf/.g.vcf.gz`, where step 24 finds step 03's) and the release in nextflow.config's manifest, as a `run pipeline_version` row of the report's run manifest (`bin/write_manifest.sh` writes the same row for `run-all.sh`). It also gets the VEP VCF (each ClinVar hit's gnomAD AF and rsIDs), CPSR's classification table and the cDNA read check of its P/LP rows, `ALIGN_MINIMAP2`'s read count, and the clinical filter's and slivar's summary tables. The slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section, run `GENOME_DIR= scripts/24-html-report.sh ` on the Nextflow output | +| HTML report (24) | renders every section from the summary `og summary collect` writes | `HTML_REPORT` runs the same code on the outputs of this run's steps, among them the cDNA read check of the ClinVar hits, the CYP2D6 depth check, the phased compound-het pairs, the RFC1 motif count, AnnotSV's class 4 and 5 table, the chrM annotation and step 35's HBA1/HBA2 count, CYP21A2 and gene-dosage tables. It also gets the sample's gVCF (DEEPVARIANT's or the `gvcf` column's, linked as `vcf/.g.vcf.gz`, where step 24 finds step 03's) and the release in nextflow.config's manifest, as a `run pipeline_version` row of the report's run manifest (`bin/write_manifest.sh` writes the same row for `run-all.sh`). It also gets the VEP VCF (each ClinVar hit's gnomAD AF and rsIDs), CPSR's classification table and the cDNA read check of its P/LP rows, `ALIGN_MINIMAP2`'s read count, and the clinical filter's and slivar's summary tables. The slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section, run `GENOME_DIR= scripts/24-html-report.sh ` on the Nextflow output | | CNVpytor (18) | mounts each resource file over the image's data folder | copies the files into the image's `site-packages`, so it needs a writable container | | Cyrius (21) | runs Cyrius from the install `setup.sh --cyrius` made | the same install, given as `--cyrius_install` (see [Cyrius, opt-in](#cyrius-opt-in)) | diff --git a/docs/run-state.md b/docs/run-state.md index 1f481ff45..8474435c5 100644 --- a/docs/run-state.md +++ b/docs/run-state.md @@ -26,7 +26,7 @@ When `run-all.sh` ends, however it ends, a `planned` row gives way to the step's `assets/steps.tsv` has one row per step: its number, its label as `run-all.sh` prints it, its `--tools` name, whether `run-all.sh` records it in `run_status.tsv`, its Nextflow processes, the report sections it feeds, the steps whose output it reads, and its usual hours. The hours are the upper end of the step's time in the [runtime table](hardware-requirements.md#runtime-per-step); `hours_from` says whether that figure was measured or is the page's estimate, and is empty when the table has no time for the step. On a measured row, `hours_low` is the lower end of the same measured time, always below `hours`; a part that runs once per reference, such as building an index, is left out of it. -`tests/test_run_state.py` keeps the catalogue true: every step `run-all.sh` plans has its row with the same label and tool, the recorded steps are the ones `bin/collect_summary.py` checks for stale results, every report section belongs to the row of its step, every process in `modules/local` belongs to a step, and the tools are the ones `main.nf` accepts. +`tests/test_run_state.py` keeps the catalogue true: every step `run-all.sh` plans has its row with the same label and tool, the recorded steps are the ones `og summary collect` checks for stale results, every report section belongs to the row of its step, every process in `modules/local` belongs to a step, and the tools are the ones `main.nf` accepts. ## What a step's state means diff --git a/docs/rust.md b/docs/rust.md index 5f0793205..0e0de2374 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -21,7 +21,7 @@ rust/ crates/og-qc/ og qc sex-loss: mosaic loss of Y or X for step 33 crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) - crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass, or one group with --only a|b|c (no caller yet); og roh summary (step 11) + crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass (render_report.py runs it for steps 24 and HTML_REPORT and generate-report.sh), or one group with --only a|b|c; og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read @@ -36,7 +36,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. @@ -247,5 +247,4 @@ The shared `og-io` reader consumes LF, CRLF and lone CR records with storage bou Step 42 uses `og traits table`, `og traits phase-sites` and `og traits check`. The readers retain only catalog sites. The trait and genotype-rule output -columns and readings stay the same. `bin/traits_report.py` keeps report -metadata until the Python summary and report collectors move. +columns and readings stay the same. diff --git a/rust/README.md b/rust/README.md index c69f31c2d..ce5fae19c 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), which steps 24 and HTML_REPORT and `generate-report.sh` write through `bin/render_report.py`; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index e55cb03d3..c6efbb918 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -27,7 +27,7 @@ - step 33's mosaic rows come from og qc sex-loss, step 26b's table and painting from og ancestry-fine report, step 35b's table from og paraphase table, step 39's from bin/couple_report.py and step 42's - from bin/traits_report.py, each fed made-up inputs; step 46's table and + from og traits table, each fed made-up inputs; step 46's table and step 35's HBA1/HBA2 table are what og cn call writes for made-up depths (HBA_TABLE and LPA_TABLE; tests/test_hba_copy_number.py and tests/test_lpa_kiv2.py check that og still writes them), step 49's depth From 17728ec4c929bc29ba1d92f7512301507efedd19 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:48:41 +0100 Subject: [PATCH 24/86] test(report): check that a run collects the summary once The fake-docker run-all case now checks that generate-report.sh renders from the summary step 24 wrote, that both reports carry one generated time, and that a step output newer than summary.json makes it collect again. Bead: pgp-bs9u.45 --- tests/fake-docker/run-all-partial-failure.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 098370891..5deda814f 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -150,6 +150,22 @@ for r in "$TXT" "$HTML"; do grep -q 'STALE: from an earlier run.*step 19 was not finished' "$r" \ || miss "${r##*/} does not mark step 19's old result stale, but Delly did not finish in this run" done +# One collect per run: generate-report.sh renders the text report from the +# summary step 24 wrote just before it, so both reports carry one time. +grep -q '^Rendering from summary.json' "${S}/logs/generate-report.log" \ + || miss "generate-report.sh collected the summary again after step 24 wrote it: $(head -3 "${S}/logs/generate-report.log")" +t_txt=$(sed -n 's/^ Generated: //p' "$TXT" | head -1) +t_html=$(grep -o 'Generated: [^<]*' "$HTML" | head -1 | sed 's/^Generated: //') +[ -n "$t_txt" ] && [ "$t_txt" = "$t_html" ] || miss "the text report says generated '${t_txt}', the HTML report '${t_html}'" +# The reverse: a step output newer than summary.json makes it collect again. +roh=$(find "$S" -name 'sample1_roh.txt' -type f | head -1) +[ -n "$roh" ] || fail "no ROH output to touch" +touch "$roh" +run_expect 0 recollect "${SCRIPTS}/generate-report.sh" sample1 +if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect.out" 2> /dev/null; then + miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" +fi +touch -d '30 days ago' "$roh" # back to its age, for the cases below # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From 2d09b4f983e1e18a9bd5d1692898b927061e05f5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:24 +0100 Subject: [PATCH 25/86] perf(summary): read the clinical and other sections one at a time og summary collect read groups b and c at once, and group c on a pool of up to eight threads, so a run's peak memory was the sum of the sections in flight. The sample VCF, the one input that takes time, stays on its own thread; every other section is read one at a time meanwhile. On a synthetic folder with every side table 50 times its demo size the peak footprint fell from 119-130 MB to 105-112 MB, CPU time unchanged. Bead: pgp-bs9u.45 --- rust/crates/og-summary/src/collect/group_b.rs | 39 ++++++++----------- rust/crates/og-summary/src/collect/group_c.rs | 39 +++++-------------- rust/crates/og-summary/src/collect/post.rs | 12 +++--- 3 files changed, 33 insertions(+), 57 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs index 394178cf5..81426f5bb 100644 --- a/rust/crates/og-summary/src/collect/group_b.rs +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -6,8 +6,8 @@ //! section, the ACMG tier, the carrier list and CPSR's call floor; the //! clinical VCF feeds the record count and the ACMG tier; AnnotSV's table //! and step 35's copy-number tables feed both two-hit readings. The -//! ClinVar section (with its VEP lookups) and the clinical section (with -//! the clinical VCF) are read on their own threads. +//! sections are read one at a time, so the peak memory is the biggest +//! section's, not the sum. use super::acmg::{self, ClinicalVcf}; use super::carrier::Carrier; @@ -128,15 +128,15 @@ pub fn read(input: &Input, folder: &Folder) -> Vec
{ }; let side = |rel: &str| one(rel.to_owned()); - let (clinvar, clinical, rest) = std::thread::scope(|scope| { - let clinvar = scope.spawn(|| -> Read { - let Some(h) = &hits else { return Ok(None) }; - let data = ClinVar::read(h, d, &side, &s)?; - with_mtime(&h.path, h.rel.clone(), Data::ClinVar(data)) - }); - let clinical = scope.spawn(|| -> Read { - let at = clinical_table.clone().or_else(|| clinical_vcf_at.clone()); - let Some((p, rel)) = at else { return Ok(None) }; + // One section at a time, so the peak memory is the biggest section's. + let clinvar: Read = match &hits { + None => Ok(None), + Some(h) => ClinVar::read(h, d, &side, &s) + .and_then(|data| with_mtime(&h.path, h.rel.clone(), Data::ClinVar(data))), + }; + let clinical: Read = match clinical_table.clone().or_else(|| clinical_vcf_at.clone()) { + None => Ok(None), + Some((p, rel)) => { let vcf = clinical_vcf_at.as_ref().map(|(v, _)| ClinicalVcf::read(v)); let tier = acmg::Inputs { assets: &input.assets, @@ -144,20 +144,15 @@ pub fn read(input: &Input, folder: &Folder) -> Vec
{ clinical: vcf.as_ref(), deletions: &deletions, }; - let data = Clinical::read( + Clinical::read( clinical_table.as_ref().map(|(p, _)| p.as_str()), vcf.as_ref(), &tier, - )?; - with_mtime(&p, rel, Data::Clinical(data)) - }); - let rest = read_rest(input, &s, index, &find, &hits, &deletions); - let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { - h.join() - .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) - }; - (join(clinvar), join(clinical), rest) - }); + ) + .and_then(|data| with_mtime(&p, rel, Data::Clinical(data))) + } + }; + let rest = read_rest(input, &s, index, &find, &hits, &deletions); let [carrier, pharmcat, cpic, pypgx, cyrius, cpsr, slivar] = rest; let reads = [ clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index d9c8c9506..edf94ecc7 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -200,36 +200,15 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { tables::mei, tables::mosaic, ]; - // The readers are independent: one thread each, over a small pool, so a - // big input (a VCF, a table of thousands of rows) never holds up the - // others. A panic stays in its section. - let jobs = std::thread::available_parallelism().map_or(4, |n| n.get().min(8)); - let next = std::sync::atomic::AtomicUsize::new(0); - let slots: Vec>> = (0..readers.len()) - .map(|_| std::sync::Mutex::new(None)) - .collect(); - std::thread::scope(|scope| { - for _ in 0..jobs { - scope.spawn(|| { - loop { - let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let Some(r) = readers.get(i) else { break }; - let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r(c))) - .unwrap_or_else(|_| Err("internal error: the reader stopped".to_owned())); - if let Ok(mut slot) = slots[i].lock() { - *slot = Some(out); - } - } - }); - } - }); - slots - .into_iter() - .map(|m| { - m.into_inner() - .ok() - .flatten() - .unwrap_or_else(|| Err("internal error: the reader stopped".to_owned())) + // One reader at a time: a run's peak memory is then the biggest single + // section, not the sum of all of them. The sample VCF, the one input + // that takes time, is read on its own thread meanwhile (`read_a`). A + // panic stays in its section. + readers + .iter() + .map(|r| { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r(c))) + .unwrap_or_else(|_| Err("internal error: the reader stopped".to_owned())) }) .collect() } diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 6065d5402..305bc87ac 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -120,14 +120,17 @@ no standard caller sees them"; const MCA_NOT_ASSESSED: &str = "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share \ of the blood cells): step 51 (MoChA) did not assess this sample"; -/// Reads the whole summary of a sample folder. The three groups are read -/// at once over one walk of the folder. An error is a run status or a -/// manifest that is there and cannot be read. +/// Reads the whole summary of a sample folder, over one walk of the +/// folder. An error is a run status or a manifest that is there and cannot +/// be read. pub fn collect(input: &Input) -> Result { let folder = Folder::open(input)?; + // Group a reads the sample VCF, the input that takes time, on its own + // thread; groups b and c read their sections meanwhile, one at a time, + // so the peak memory stays near the biggest section's. let (a, b, mut c) = std::thread::scope(|scope| { let a = scope.spawn(|| read_a(input, &folder)); - let b = scope.spawn(|| group_b::read(input, &folder)); + let b = group_b::read(input, &folder); let c = read_c(input, &folder); let stopped = |specs: &[super::model::Spec]| -> Vec
{ specs @@ -142,7 +145,6 @@ pub fn collect(input: &Input) -> Result { .collect() }; let a = a.join().unwrap_or_else(|_| stopped(&super::model::GROUP_A)); - let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); (a, b, c) }); let declared_sex = declared(input, &folder, qc_declared(&a)); From 7c06808cada67696c24232b5675bcf59aa5468bd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:24 +0100 Subject: [PATCH 26/86] refactor(report): delete run_delta.py, which only the old collector imported og run delta makes the comparison and og summary collect writes New since. The two run-delta checks of test_collect_summary.py 21b now run og run delta; test_run_eta.py no longer checks the old command line. Bead: pgp-bs9u.45 --- bin/owngenome_live_recheck.py | 3 +- bin/run_delta.py | 288 ---------------------------------- docs/hardware-requirements.md | 2 + scripts/ci/e2e-case-paths.tsv | 4 +- tests/test_collect_summary.py | 27 +++- tests/test_render_report.py | 2 +- tests/test_run_eta.py | 4 +- 7 files changed, 31 insertions(+), 299 deletions(-) delete mode 100755 bin/run_delta.py diff --git a/bin/owngenome_live_recheck.py b/bin/owngenome_live_recheck.py index fed062fdb..5b235d76a 100644 --- a/bin/owngenome_live_recheck.py +++ b/bin/owngenome_live_recheck.py @@ -52,7 +52,8 @@ def recheck_state(run_dir, state, events): except (OSError, ValueError, AttributeError): return {} # A check older than the run's run_manifest.tsv describes an earlier run: what it calls newer may - # be what this run used. Not shown, as in the reports (bin/run_delta.py read_recheck). + # be what this run used. Not shown, as in the reports (og run delta, read_recheck + # in rust/crates/og-runlog/src/delta.rs). if utc14(manifest_written(run_dir)) > utc14(st.get("checked_utc")): return {} steps = {} diff --git a/bin/run_delta.py b/bin/run_delta.py deleted file mode 100755 index 8b7d51ca0..000000000 --- a/bin/run_delta.py +++ /dev/null @@ -1,288 +0,0 @@ -#!/usr/bin/env python3 -"""run_delta.py: what changed since the previous run of the same sample. - -The command is `og run delta` (rust/crates/og-runlog, docs/delta-report.md): -scripts/generate-report.sh runs it after the text report, before it keeps a -copy of this run's summary.json and run_manifest.tsv under -/history//, and it writes .txt, .html and -.json. This module is the same comparison for bin/collect_summary.py, -whose "New since" badges need it. Standard library only. - -What it compares: - inputs the run manifest (bin/write_manifest.sh): versions.env values, the - image each *_IMAGE names and the digest it resolved to, the data - rows (ClinVar file date, VEP cache, CPSR bundle, pypgx bundle, HLA - database, ancestry panel, PGS headers), the pipeline commit and the - declared sex. When and by which script it was written are left out. - sections a report section that ran in one run and not in the other. - ClinVar hits added or dropped (by chrom:pos ref>alt), a hit whose - significance changed (reclassified), whose review status or stars - changed (re-starred), or whose genotype changed. - CPSR the count of each classification. - PGx each CPIC gene's diplotype and phenotype, pypgx's CYP2D6 call and - the CYP2D6 consensus. - PRS scores added or dropped, and a percentile that moved by - PRS_BAND points or more, or appeared or went away. -For each result change it lists the changed inputs that feed that section, or -says that the inputs it tracks are the same and the call itself differs. -changed_sections() names the summary sections the changes touch, and -read_recheck() the last database re-check (recheck/state.json beside the -summary, bin/owngenome_recheck.py). -""" -import json -import os - -PRS_BAND = 5.0 # percentile points - -# Changed inputs that can explain a change in a section: an input whose name -# contains one of these (case-insensitive). The pipeline commit explains any. -WHY_TOKENS = { - "clinvar": ["clinvar", "DEEPVARIANT", "BCFTOOLS", "MINIMAP2", "BWA", "GATK", "FASTP"], - "cpsr": ["PCGR", "VEP", "DEEPVARIANT"], - "cpic": ["PHARMCAT", "PYPGX", "CYRIUS", "DEEPVARIANT", "BCFTOOLS"], - "pypgx": ["PYPGX", "pypgx_bundle"], - "cyp2d6": ["PYPGX", "pypgx_bundle", "CYRIUS", "PHARMCAT"], - "prs": ["pgs:", "PGSC", "PLINK2", "ancestry_panel", "DEEPVARIANT"], -} -IGNORED_RUN_KEYS = ("written_utc", "written_by") -# The summary section of each result area (diff_states names its own). -SECTION_OF_AREA = {"ClinVar": "clinvar", "CPSR": "cpsr", "PGx (CPIC)": "cpic", "PGx (pypgx)": "pypgx", - "PGx (CYP2D6 consensus)": "pypgx", "PRS": "prs"} - - -def read_manifest(path): - """{(section, key): value} of a run_manifest.tsv; {} when there is none.""" - rows = {} - if not path or not os.path.isfile(path): - return rows - with open(path, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 3 and not c[0].startswith("#"): - rows[(c[0], c[1])] = "\t".join(c[2:]) - return rows - - -def manifest_items(rows): - """The manifest as {item: value}, one entry per thing that can change. - An image is keyed by its *_IMAGE name, its value is 'ref (digest)', so a - new tag and a new digest of the same tag are each one change.""" - out = {} - for (sec, key), val in rows.items(): - if sec == "run": - if key not in IGNORED_RUN_KEYS: - out[key] = val - elif sec == "version": - if key.endswith("_IMAGE"): - out[key] = f"{val} ({rows.get(('image', val), 'not in the manifest')})" - else: - out[key] = val - elif sec == "data": - out[key] = val - return out - - -def change(area, kind, item, before, after, why=None, section=None): - c = {"area": area, "kind": kind, "item": item, "before": before, "after": after} - if why is not None: - c["why"] = why - if section is not None: - c["section"] = section # the summary.json section it changes: the report marks it "new since" - return c - - -def diff_inputs(prev, cur): - a, b = manifest_items(prev), manifest_items(cur) - out = [] - for k in sorted(set(a) | set(b)): - if a.get(k) != b.get(k): - out.append(change("inputs", "input", k, a.get(k, "absent"), b.get(k, "absent"))) - return out - - -def why_for(area, inputs): - tokens = [t.lower() for t in WHY_TOKENS.get(area, [])] + ["git_commit"] - hit = [c for c in inputs if any(t in c["item"].lower() for t in tokens)] - if not hit: - return "the inputs this delta tracks for it are the same; the call itself differs" - return "changed inputs: " + "; ".join(f"{c['item']} {c['before']} -> {c['after']}" for c in hit) - - -def section(s, key): - return ((s.get("sections") or {}).get(key)) or {} - - -def has_data(sec): - return sec.get("state") in ("ok", "stale") - - -def diff_states(p, c): - out = [] - keys = sorted(set(p.get("sections") or {}) | set(c.get("sections") or {})) - for k in keys: - a, b = section(p, k), section(c, k) - # A section only one summary knows (a newer version added it) and that did not run reads as - # missing on both sides, so a pipeline upgrade lists no noise rows. - sa, sb = a.get("state", "missing"), b.get("state", "missing") - if sa != sb: - out.append(change(b.get("title") or a.get("title") or k, "section", k, sa, sb, section=k)) - return out - - -def hit_key(h): - return f"{h.get('chrom')}:{h.get('pos')} {h.get('ref')}>{h.get('alt')}" - - -def diff_clinvar(p, c): - a, b = section(p, "clinvar"), section(c, "clinvar") - if not (has_data(a) and has_data(b)): - return [] - ha = {hit_key(h): h for h in a.get("data", {}).get("hits", [])} - hb = {hit_key(h): h for h in b.get("data", {}).get("hits", [])} - out = [] - for k in sorted(set(ha) | set(hb)): - x, y = ha.get(k), hb.get(k) - gene = (y or x).get("gene", ".") - item = f"{k} ({gene})" - if x is None: - out.append(change("ClinVar", "added", item, "absent", f"{y.get('significance')}, {y.get('stars')} stars")) - elif y is None: - out.append(change("ClinVar", "dropped", item, f"{x.get('significance')}, {x.get('stars')} stars", "absent")) - else: - if x.get("significance") != y.get("significance"): - out.append(change("ClinVar", "reclassified", item, x.get("significance"), y.get("significance"))) - if x.get("stars") != y.get("stars") or x.get("review_status") != y.get("review_status"): - out.append(change("ClinVar", "re-starred", item, - f"{x.get('stars')} stars ({x.get('review_status')})", - f"{y.get('stars')} stars ({y.get('review_status')})")) - if x.get("genotype") != y.get("genotype"): - out.append(change("ClinVar", "genotype", item, x.get("genotype"), y.get("genotype"))) - return out - - -def diff_cpsr(p, c): - a, b = section(p, "cpsr"), section(c, "cpsr") - if not (has_data(a) and has_data(b)): - return [] - ca, cb = a.get("data", {}).get("classification") or {}, b.get("data", {}).get("classification") or {} - return [change("CPSR", "classification count", k, ca.get(k, 0), cb.get(k, 0)) - for k in sorted(set(ca) | set(cb)) if ca.get(k, 0) != cb.get(k, 0)] - - -def diff_cpic(p, c): - a, b = section(p, "cpic"), section(c, "cpic") - if not (has_data(a) and has_data(b)): - return [] - ga = {g.get("gene"): g for g in a.get("data", {}).get("genes", [])} - gb = {g.get("gene"): g for g in b.get("data", {}).get("genes", [])} - out = [] - for k in sorted(set(ga) | set(gb), key=str): - x, y = ga.get(k), gb.get(k) - fx = f"{x.get('diplotype')} ({x.get('phenotype')})" if x else "absent" - fy = f"{y.get('diplotype')} ({y.get('phenotype')})" if y else "absent" - if fx != fy: - out.append(change("PGx (CPIC)", "call", k, fx, fy)) - return out - - -def diff_cyp2d6(p, c): - out = [] - a, b = section(p, "pypgx"), section(c, "pypgx") - if has_data(a) and has_data(b): - x, y = a.get("data", {}).get("cyp2d6"), b.get("data", {}).get("cyp2d6") - if x != y: - out.append(change("PGx (pypgx)", "call", "CYP2D6", x, y)) - x = ((p.get("cyp2d6") or {}).get("consensus") or {}).get("result") - y = ((c.get("cyp2d6") or {}).get("consensus") or {}).get("result") - if x != y: - out.append(change("PGx (CYP2D6 consensus)", "call", "CYP2D6", x or "none", y or "none")) - return out - - -def pct(v): - try: - return float(v) - except (TypeError, ValueError): - return None - - -def shown(v): - return "none" if v is None else v - - -def diff_prs(p, c): - a, b = section(p, "prs"), section(c, "prs") - if not (has_data(a) and has_data(b)): - return [] - sa = {r.get("pgs_id"): r for r in a.get("data", {}).get("scores", [])} - sb = {r.get("pgs_id"): r for r in b.get("data", {}).get("scores", [])} - out = [] - for k in sorted(set(sa) | set(sb), key=str): - x, y = sa.get(k), sb.get(k) - item = f"{k} ({(y or x).get('condition', '')})" - if x is None or y is None: - out.append(change("PRS", "added" if x is None else "dropped", item, - "absent" if x is None else "scored", "scored" if x is None else "absent")) - continue - px, py = pct(x.get("percentile")), pct(y.get("percentile")) - if (px is None) != (py is None) or (px is not None and abs(px - py) >= PRS_BAND): - # A percentile of 0 is a value: only a missing one reads "none". - out.append(change("PRS", "percentile", item, shown(x.get("percentile")), shown(y.get("percentile")))) - return out - - -def compare(prev_summary, prev_manifest, summary, manifest): - inputs = diff_inputs(prev_manifest, manifest) - results = [] - for area, fn in (("clinvar", diff_clinvar), ("cpsr", diff_cpsr), ("cpic", diff_cpic), - ("cyp2d6", diff_cyp2d6), ("prs", diff_prs)): - for ch in fn(prev_summary, summary): - ch["why"] = why_for(area, inputs) - ch["section"] = SECTION_OF_AREA.get(ch["area"], area) - results.append(ch) - return inputs + diff_states(prev_summary, summary) + results - - -def changed_sections(changes): - """The summary sections a delta changes, in order: what the report marks 'New since'.""" - return list(dict.fromkeys(c["section"] for c in changes if c.get("section"))) - - -def newest_history(sample_dir): - """The newest history// folder with a summary.json, or None.""" - h = os.path.join(sample_dir, "history") - try: - names = sorted(n for n in os.listdir(h) if os.path.isfile(os.path.join(h, n, "summary.json"))) - except OSError: - return None - return os.path.join(h, names[-1]) if names else None - - -def read_recheck(path): - """/recheck/state.json (bin/owngenome_recheck.py) as a short block, or None.""" - try: - with open(path) as f: - st = json.load(f) - except (OSError, ValueError): - return None - if not isinstance(st, dict) or not isinstance(st.get("items"), list): - return None - # A check of an earlier run: /run_manifest.tsv was written after it, so its "newer than - # the run used" may already be what this run used. Not shown; the next check replaces it. - man = read_manifest(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(path))), "run_manifest.tsv")) - if utc14(man.get(("run", "written_utc"))) > utc14(st.get("checked_utc")): - return None - return {"checked_utc": st.get("checked_utc"), - "items": [{k: i.get(k) for k in ("item", "label", "class", "ran_with", "upstream", "action")} - for i in st["items"] if isinstance(i, dict)], - "affected_steps": st.get("affected_steps") or [], - "rerun": st.get("rerun") if isinstance(st.get("rerun"), dict) else None} - - -def utc14(text): - """'2026-09-02 12:00:00 UTC' or '2026-09-02T12:00:00Z' as '20260902120000'; '' without one.""" - return "".join(c for c in str(text or "") if c.isdigit())[:14] - - -if __name__ == "__main__": - raise SystemExit("bin/run_delta.py has no command line any more: run bin/og run delta (docs/delta-report.md)") diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index ba4ef1c61..67a7b09ee 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -256,6 +256,8 @@ Both ends of each measured figure are also in `assets/steps.tsv` (`hours_low` an So the reports cost about 25 s per sample, not 1-3 minutes, and almost all of it is collecting the summary, which each of the two scripts does once. Under `cProfile` (one HTML run), 82% of the collection was `sec_variants` reading every record of the sample's small-variant VCF in Python, and 9% the ROH section. Rendering is under half a second. The HTML report was 2.0-2.7 MiB and the search index 8.3-12.2 MiB. +Since then the Rust `og summary collect` writes `summary.json`, and `generate-report.sh` renders from the `summary.json` step 24 wrote unless a step output is newer than it, so a run collects each sample once. `run_delta.py` is gone: `og run delta` makes the comparison. + The command, per genome (`` holds the sample folder, `` is empty and writable, run from the repo root): ```sh diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 50dd13368..f4dd8e3cd 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -159,7 +159,7 @@ setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv asse short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh @@ -175,7 +175,7 @@ vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_gene vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 99be691f1..20eb59617 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -82,7 +82,7 @@ frequency caveat on every hit; 21b. each hit carries the rsIDs of VEP's Existing_variation ('rs123&COSV1' gives ['rs123']; another allele's ids are not taken), [] with no VEP record or no VEP file, and the run delta - (bin/run_delta.py) reads the same with or without them; + (og run delta) reads the same with or without them; 22. CPSR beside ClinVar (step 17): the P/LP rows by final or own class, the rows where CLINVAR_CLASSIFICATION and CPSR_CLASSIFICATION disagree on P/LP with CPSR's ACMG codes, the final P/LP rows CPSR's own rules class @@ -729,8 +729,24 @@ def clinvar_quality(work, schema): def clinvar_rsids(work, schema): """21b. the rsIDs of each ClinVar hit, from VEP's Existing_variation.""" - import run_delta d = os.path.join(work, "clinvar-rsids", "S") + + def changes(prev, cur): + """og run delta's changes from summary PREV to summary CUR.""" + w = os.path.join(work, "clinvar-rsids-delta") + shutil.rmtree(w, ignore_errors=True) + for name, summary in (("prev", prev), ("cur", cur)): + os.makedirs(f"{w}/{name}") + with open(f"{w}/{name}/summary.json", "w") as f: + json.dump(summary, f) + p = subprocess.run([collect_summary.og(), "run", "delta", "--sample", "S", "--summary", + f"{w}/cur/summary.json", "--out", f"{w}/delta", "--previous", f"{w}/prev"], + capture_output=True, text=True) + if p.returncode != 0: + return p.stderr + with open(f"{w}/delta.json") as f: + return json.load(f)["changes"] + put(f"{d}/clinvar/S_clinvar_hits.vcf", FLOOR_HITS) vep_head = "##fileformat=VCFv4.2\n" + CSQ_EV_HEAD + VCF_HEAD.split("\n", 1)[1] write_tabixed_vcf(f"{d}/vep/S_vep.vcf.gz", vep_head, [ @@ -748,12 +764,13 @@ def clinvar_rsids(work, schema): bare = json.loads(json.dumps(summ)) for h in bare["sections"]["clinvar"]["data"]["hits"]: h.pop("rsids") - check("ClinVar rsIDs: the run delta reads the same with or without them", - run_delta.compare(bare, {}, summ, {}) == run_delta.compare(summ, {}, summ, {}), "") + got = changes(bare, summ) + check("ClinVar rsIDs: the run delta reads the same with or without them", got == [], got) moved = json.loads(json.dumps(summ)) moved["sections"]["clinvar"]["data"]["hits"][0]["significance"] = "Benign" + got = changes(bare, moved) check("ClinVar rsIDs: red control, the run delta does see a reclassified hit", - run_delta.compare(bare, {}, moved, {}) != run_delta.compare(summ, {}, summ, {}), "") + isinstance(got, list) and any(c["kind"] == "reclassified" for c in got), got) shutil.rmtree(f"{d}/vep") none = collect_summary.collect("S", d)["sections"]["clinvar"]["data"]["hits"] check("ClinVar rsIDs: without step 13 every hit has [] and the section still reads", diff --git a/tests/test_render_report.py b/tests/test_render_report.py index 1d85a1758..36add70fe 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -405,7 +405,7 @@ def chapter_checks(work): check("chapters: red control: a dropped card is caught", sorted(re.findall(r"

([^<]*)

", html.replace("

Traits

", ""))) != GOLDEN_TITLES) - # "New since" and the database re-check (bin/owngenome_recheck.py, bin/run_delta.py through collect_summary) + # "New since" and the database re-check (bin/owngenome_recheck.py, og summary collect) since = json.loads(json.dumps(summ)) since["new_since"] = {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["clinvar"]} since["recheck"] = {"checked_utc": "2026-02-02T00:00:00Z", "affected_steps": ["06", "24"], "rerun": None, diff --git a/tests/test_run_eta.py b/tests/test_run_eta.py index ded2e7e0b..f3cc0a6ca 100644 --- a/tests/test_run_eta.py +++ b/tests/test_run_eta.py @@ -26,7 +26,7 @@ 15. without --sample, the trace of a run that run-all.sh has not finished is left out of the step times, and counts once run_status.tsv says it finished; a run_status.tsv that cannot be read leaves out the newest trace, never a crash. - 16. bin/run_eta.py, run_state.py, run_headroom.py and run_delta.py, run as commands, + 16. bin/run_eta.py, run_state.py and run_headroom.py, run as commands, exit 1 and name their og run command, never a silent exit 0. Each check is then run against a planted mistake (a mutant of run_eta.py or a broken config) and must fail there: a check that cannot fail is no check. @@ -293,7 +293,7 @@ def unreadable(path): return out -OLD_CLIS = {"run_eta.py": "eta", "run_state.py": "state", "run_headroom.py": "headroom", "run_delta.py": "delta"} +OLD_CLIS = {"run_eta.py": "eta", "run_state.py": "state", "run_headroom.py": "headroom"} def old_cli_refuses(path, cmd): From bf525e2d2be2cc2118e5dd0551467b1dbc2e4ae0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:39 +0100 Subject: [PATCH 27/86] docs(changelog): og writes the summary the reports read, once per run Bead: pgp-bs9u.45 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bf731b53..2a462b156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again only when a file it reads is newer than that summary.json. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: + - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. + - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. + - The Rust `og` binary now writes the whole summary.json: `og summary collect` without `--only` reads all 48 sections over one walk of the sample folder, then runs the post pass. The post pass joins sections: the repeat readings with the traits and CPIC states, Locityper's primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter with the declared sex. It also builds the CYP2D6 block and writes `not_run`, `not_assessed`, `recheck` and `new_since`. Without `--out` it writes `summary.json` in the sample folder, as the Python did; `--only` still needs `--out`. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The whole file is the same as the Python's on 8 of 9 new parity cases, compared as JSON with key order and only `generated_utc` masked; the ninth holds the deliberate fix below. On a synthetic folder with every section it takes 0.01 to 0.03 s and 15 MB, where the Python took 0.15 to 0.21 s and 42 to 44 MB. With a 1,000,000-record VCF it takes 0.44 to 0.62 s and 17 to 20 MB, where the Python took 1.5 s and 42 to 44 MB. What differs from the Python: - The post pass keeps the summary. The Python ran it outside its per-section error handling, so one row of an unexpected shape there lost summary.json and both reports. The Rust parts read with defaults and have no such error to raise; a part that returns an error makes only its own section unreadable, with a note that names the file. - A section reader that stops on an error the Python did not catch keeps the summary too. A telomere table with an infinite read length stopped the Python with an OverflowError and no summary.json; og marks only the telomere section unreadable. From 84027ab0e4f670e867c6b60c9bba31cdee4725b5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:41:38 +0100 Subject: [PATCH 28/86] test(report): read the HTML report's generated time up to its separator The HTML header puts the latest run after the generated time on the same line, so the comparison with the text report took both. Bead: pgp-bs9u.45 --- tests/fake-docker/run-all-partial-failure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 5deda814f..556872618 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -155,7 +155,7 @@ done grep -q '^Rendering from summary.json' "${S}/logs/generate-report.log" \ || miss "generate-report.sh collected the summary again after step 24 wrote it: $(head -3 "${S}/logs/generate-report.log")" t_txt=$(sed -n 's/^ Generated: //p' "$TXT" | head -1) -t_html=$(grep -o 'Generated: [^<]*' "$HTML" | head -1 | sed 's/^Generated: //') +t_html=$(grep -o 'Generated: [^<&]*' "$HTML" | head -1 | sed 's/^Generated: //; s/ *$//') [ -n "$t_txt" ] && [ "$t_txt" = "$t_html" ] || miss "the text report says generated '${t_txt}', the HTML report '${t_html}'" # The reverse: a step output newer than summary.json makes it collect again. roh=$(find "$S" -name 'sample1_roh.txt' -type f | head -1) From 0aab619e656787283ae8ef042520bcc53c248635 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:45:00 +0100 Subject: [PATCH 29/86] fix(report): collect again after a removed file, and keep the temporary summary private generate-report.sh rendered from step 24's summary.json when no file was newer, so a step output removed after the collect went unnoticed. A subfolder newer than the summary (other than logs/) now means collect again, and the fake-docker run-all case checks it. collect_summary.collect() without OUT gave og a file in the shared temporary folder, and og's rename over it dropped mkstemp's owner-only mode. It now uses a private temporary folder. Bead: pgp-bs9u.45 --- CHANGELOG.md | 2 +- bin/collect_summary.py | 12 +++++------- scripts/generate-report.sh | 7 +++++-- tests/fake-docker/run-all-partial-failure.sh | 9 +++++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a462b156..f60e0c1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again only when a file it reads is newer than that summary.json. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 55bffce51..e0f524e66 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -19,6 +19,7 @@ import json import os import re +import shutil import subprocess import tempfile import zlib @@ -97,9 +98,9 @@ def collect(sample, sample_dir, declared_sex=None, out=None): written).""" tmp = None if out is None: - fd, tmp = tempfile.mkstemp(prefix="summary.", suffix=".json") - os.close(fd) - out = tmp + # A private folder: og writes a sibling file and renames it over OUT. + tmp = tempfile.mkdtemp(prefix="summary.") + out = os.path.join(tmp, "summary.json") cmd = [og(), "summary", "collect", "--sample", sample, "--sample-dir", sample_dir, "--assets", ASSETS_DIR, "--out", out] if declared_sex: @@ -118,10 +119,7 @@ def collect(sample, sample_dir, declared_sex=None, out=None): return json.load(f) finally: if tmp: - try: - os.unlink(tmp) - except OSError: - pass + shutil.rmtree(tmp, ignore_errors=True) # ClinVar review status -> stars. og clinvar hits (rust/crates/og-clinvar) holds the same table diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index 5e5d3161d..4c4415538 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -41,11 +41,14 @@ fi # the summary or beside it (the reports, the search index, the charts, the # delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch # folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a -# summary that is not there, means collect again. +# summary that is not there, means collect again. A subfolder newer than the +# summary (other than logs/) had a file added or removed since: collect again. summary_current() { local s="${SAMPLE_DIR}/summary.json" newer [ -s "$s" ] && [ ! -L "$s" ] || return 1 - newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune -o ! -type d -newer summary.json \ + newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune \ + -o -type d ! -path . ! -path ./logs -newer summary.json -print -quit \ + -o ! -type d -newer summary.json \ ! \( -path './logs/*' ! -path ./logs/run_status.tsv \) \ ! -name "${SAMPLE}_report.html" ! -name "${SAMPLE}_report.txt" ! -name "${SAMPLE}_search_index.json" \ ! -name "${SAMPLE}_charts.json" ! -name "${SAMPLE}_delta.txt" ! -name "${SAMPLE}_delta.html" \ diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 556872618..c60a3df9e 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -166,6 +166,15 @@ if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect.out" 2> /dev/n miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" fi touch -d '30 days ago' "$roh" # back to its age, for the cases below +# A file removed after the collect makes it collect again too. +extra="${roh%/*}/sample1_extra.txt" +: > "$extra" +run_expect 0 recollect-added "${SCRIPTS}/generate-report.sh" sample1 +rm -f "$extra" +run_expect 0 recollect-removed "${SCRIPTS}/generate-report.sh" sample1 +if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect-removed.out" 2> /dev/null; then + miss "generate-report.sh rendered from summary.json, but a file was removed after it was written" +fi # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From 686a0c644ce9e46541d9d527caf50f09ca5fd935 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:15:05 +0100 Subject: [PATCH 30/86] fix(report): collect again when a section's whole folder is gone Removing a step folder after the collect changes only the sample folder's own time, which the reports change too, so generate-report.sh rendered the removed results from summary.json. It now also collects again when a file a section names as its source is gone. The fake-docker run-all case checks an unchanged folder, a newer output, a removed file and a removed folder, each from a summary.json that is the newest file. Bead: pgp-bs9u.45 --- CHANGELOG.md | 2 +- scripts/generate-report.sh | 11 +++++-- tests/fake-docker/run-all-partial-failure.sh | 33 ++++++++++++++------ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f60e0c1db..1eea6ada4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source is gone. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index 4c4415538..f035ece08 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -42,10 +42,17 @@ fi # delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch # folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a # summary that is not there, means collect again. A subfolder newer than the -# summary (other than logs/) had a file added or removed since: collect again. +# summary (other than logs/) had a file added or removed since, and a section's +# source that is gone (its file or its whole folder removed) means the same: +# collect again. summary_current() { - local s="${SAMPLE_DIR}/summary.json" newer + local s="${SAMPLE_DIR}/summary.json" newer src [ -s "$s" ] && [ ! -L "$s" ] || return 1 + # Each section's "source", at the sections' depth of og's one-space indent. + while IFS= read -r src; do + case $src in *\\*) return 1 ;; esac + [ -e "${SAMPLE_DIR}/${src}" ] || return 1 + done < <(sed -n 's/^ "source": "\([^"]*\)",\{0,1\}$/\1/p' "$s") newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune \ -o -type d ! -path . ! -path ./logs -newer summary.json -print -quit \ -o ! -type d -newer summary.json \ diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index c60a3df9e..ea8d7b30c 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -157,24 +157,39 @@ grep -q '^Rendering from summary.json' "${S}/logs/generate-report.log" \ t_txt=$(sed -n 's/^ Generated: //p' "$TXT" | head -1) t_html=$(grep -o 'Generated: [^<&]*' "$HTML" | head -1 | sed 's/^Generated: //; s/ *$//') [ -n "$t_txt" ] && [ "$t_txt" = "$t_html" ] || miss "the text report says generated '${t_txt}', the HTML report '${t_html}'" -# The reverse: a step output newer than summary.json makes it collect again. +# The reverse: a change after summary.json was written makes it collect +# again. Each check first makes summary.json the newest file, as step 24 +# leaves it (the run's history/ copy came after it), and waits a second, so +# only the change it makes is newer. +fresh() { touch "${S}/summary.json" && /bin/sleep 1; } +rendered_from() { grep -q '^Rendering from summary.json' "${CASE_WORK}/$1.out" 2> /dev/null; } +fresh +run_expect 0 current "${SCRIPTS}/generate-report.sh" sample1 +rendered_from current || miss "generate-report.sh collected again, but nothing changed after summary.json was written" +# A step output newer than summary.json. roh=$(find "$S" -name 'sample1_roh.txt' -type f | head -1) [ -n "$roh" ] || fail "no ROH output to touch" +fresh touch "$roh" run_expect 0 recollect "${SCRIPTS}/generate-report.sh" sample1 -if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect.out" 2> /dev/null; then - miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" -fi +! rendered_from recollect || miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" touch -d '30 days ago' "$roh" # back to its age, for the cases below -# A file removed after the collect makes it collect again too. +# A file removed: its folder is newer than summary.json. extra="${roh%/*}/sample1_extra.txt" : > "$extra" -run_expect 0 recollect-added "${SCRIPTS}/generate-report.sh" sample1 +fresh rm -f "$extra" run_expect 0 recollect-removed "${SCRIPTS}/generate-report.sh" sample1 -if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect-removed.out" 2> /dev/null; then - miss "generate-report.sh rendered from summary.json, but a file was removed after it was written" -fi +! rendered_from recollect-removed || miss "generate-report.sh rendered from summary.json, but a file was removed after it was written" +# A section's whole folder removed: only the sample folder itself is newer, +# so the section's missing source must tell. +src=$(sed -n 's|^ "source": "\([^"/]*\)/.*|\1|p' "${S}/summary.json" | head -1) +[ -n "$src" ] && [ -d "${S}/${src}" ] || fail "summary.json names no section folder: $(grep -m3 '"source"' "${S}/summary.json")" +fresh +mv "${S}/${src}" "${CASE_WORK}/moved-away" +run_expect 0 recollect-folder "${SCRIPTS}/generate-report.sh" sample1 +mv "${CASE_WORK}/moved-away" "${S}/${src}" +! rendered_from recollect-folder || miss "generate-report.sh rendered from summary.json, but ${src}/, a section's folder, was removed after it was written" # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From eaf6bfc6b5e6874b47310eb5d5592c562ca5dc23 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:16:18 +0100 Subject: [PATCH 31/86] ci(guard): read the SMN1 marker from the test's own table The traits control reached the marker through traits_report, which this branch deletes; test_traits_report.py holds MARKERS itself now. Bead: pgp-bs9u.45 --- .github/workflows/guard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 878d9be5e..a2aee4fac 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -663,7 +663,7 @@ jobs: assert rc == 1 and test["FAILS"] == 1 assert f"[FAIL] {rid}, reference blocks only: {label}" in log.getvalue() print(f"Control: {rid} WRONG_REFERENCE failed its independent expectation") - marker = test["tr"].MARKERS["SMN1_27134"] + marker = test["MARKERS"]["SMN1_27134"] rs, chrom, pos, ref, alt = marker["dbsnp"] marker["dbsnp"] = rs, chrom, pos + 1, ref, alt test["FAILS"] = 0 From 7aaf3a940170d18f3a94a46d7672e50caa480af1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 22:34:31 +0100 Subject: [PATCH 32/86] fix(report): collect again when the run status or a bin or assets table is gone The summary records that it read logs/run_status.tsv, which the subfolder test leaves out with logs/, and the tables in bin/ and assets/ are no section's source. generate-report.sh now collects again when that run status is gone, or when bin/, bin/.og/ or assets/ is newer than the summary. The fake-docker run-all case checks both. The CHANGELOG says the shared generated time holds only when the text report renders from step 24's summary. Bead: pgp-bs9u.45 --- CHANGELOG.md | 2 +- scripts/generate-report.sh | 13 ++++++++++--- tests/fake-docker/run-all-partial-failure.sh | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfbcdeace..ae6213999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source is gone. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index f035ece08..5bc3177b6 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -42,12 +42,16 @@ fi # delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch # folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a # summary that is not there, means collect again. A subfolder newer than the -# summary (other than logs/) had a file added or removed since, and a section's -# source that is gone (its file or its whole folder removed) means the same: -# collect again. +# summary (other than logs/) had a file added or removed since, and so did +# bin/, bin/.og/ or assets/ when it is newer. A section's source that is gone +# (its file or its whole folder removed), or the run status the summary read, +# means the same: collect again. summary_current() { local s="${SAMPLE_DIR}/summary.json" newer src [ -s "$s" ] && [ ! -L "$s" ] || return 1 + if grep -q '^ "status_file": true' "$s"; then + [ -e "${SAMPLE_DIR}/logs/run_status.tsv" ] || return 1 + fi # Each section's "source", at the sections' depth of og's one-space indent. while IFS= read -r src; do case $src in *\\*) return 1 ;; esac @@ -61,6 +65,9 @@ summary_current() { ! -name "${SAMPLE}_charts.json" ! -name "${SAMPLE}_delta.txt" ! -name "${SAMPLE}_delta.html" \ ! -name "${SAMPLE}_delta.json" -print -quit) || return 1 [ -z "$newer" ] || return 1 + newer=$(find "${PGP_ROOT}/bin" "${PGP_ROOT}/bin/.og" "${PGP_ROOT}/assets" -maxdepth 0 -newer "$s" -print -quit 2> /dev/null) \ + || return 1 + [ -z "$newer" ] || return 1 newer=$(find "${PGP_ROOT}/bin" "${PGP_ROOT}/bin/.og" "${PGP_ROOT}/assets" -maxdepth 1 ! -type d -newer "$s" -print -quit 2> /dev/null) \ || return 1 [ -z "$newer" ] diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index ea8d7b30c..00a53e4dc 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -190,6 +190,21 @@ mv "${S}/${src}" "${CASE_WORK}/moved-away" run_expect 0 recollect-folder "${SCRIPTS}/generate-report.sh" sample1 mv "${CASE_WORK}/moved-away" "${S}/${src}" ! rendered_from recollect-folder || miss "generate-report.sh rendered from summary.json, but ${src}/, a section's folder, was removed after it was written" +# The run status the summary read, removed. +grep -q '^ "status_file": true' "${S}/summary.json" || fail "summary.json read no run status" +fresh +mv "${S}/logs/run_status.tsv" "${CASE_WORK}/run_status.away" +run_expect 0 recollect-status "${SCRIPTS}/generate-report.sh" sample1 +mv "${CASE_WORK}/run_status.away" "${S}/logs/run_status.tsv" +! rendered_from recollect-status || miss "generate-report.sh rendered from summary.json, but logs/run_status.tsv was removed after it was written" +# A file removed from assets/ (a table the collect reads, as far as the check +# can tell): assets/ is newer than summary.json. +extra="${REPO_ROOT}/assets/.fake-docker-extra" +: > "$extra" +fresh +rm -f "$extra" +run_expect 0 recollect-asset "${SCRIPTS}/generate-report.sh" sample1 +! rendered_from recollect-asset || miss "generate-report.sh rendered from summary.json, but a file was removed from assets/ after it was written" # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From b55093ddf5b8fc3181913bc0ba138e28c9c6a9e5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:02:43 +0100 Subject: [PATCH 33/86] test(roh): require chromosome scope and neutral UPD hints Bead: lin-10 Bead: lin-13 Bead: lin-15 Bead: lin-16 Bead: lin-17 Bead: lic-41 --- .../og-summary/src/collect/group_c/tests.rs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 82afd2dc8..b072af7cb 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -223,3 +223,79 @@ fn every_row_of_the_repeat_ranges_table_is_read() { assert_eq!(read, loci); assert!(loci.len() >= 3, "{loci:?}"); } + +// i411-roh-scope: ST rows describe assessed chromosomes, including ones +// with no long run. A run list alone cannot establish whole-genome scope. +fn roh_scope_summary(sites: &[u8], runs: &str) -> Value { + let d = Dir::new("i411-roh-scope"); + let mut body = String::new(); + for c in sites { + body.push_str(&format!("ST\tS\tchr{c}\t40000000\t0\t60\n")); + } + body.push_str(runs); + d.put("S/roh/S_roh.txt", body); + at(&summary(&d, None), &["sections", "roh", "data"]).clone() +} + +#[test] +fn roh_scope_chr20_uses_its_own_unmasked_denominator() { + let r = roh_scope_summary(&[20], "RG\tS\tchr20\t1000001\t22000000\t21000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.3515)); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + assert_eq!(at(&r, &["upd_flag"]), &Value::Null); + assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(59_744_167)); + assert_eq!(at(&r, &["scope", "autosomes"]), &Value::Array(vec![Value::Str("chr20".into())])); +} + +#[test] +fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { + let r = roh_scope_summary(&[22, 20, 20], "RG\tS\tchr20\t1000001\t9000000\t8000000\nRG\tS\tchr22\t20000001\t28000000\t8000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1718)); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(93_162_635)); + let no_runs = roh_scope_summary(&[20], ""); + assert_eq!(at(&no_runs, &["f_roh"]), &Value::Float(0.0)); + assert_eq!(at(&no_runs, &["relationship"]), &Value::Null); +} + +#[test] +fn roh_scope_sparse_all_autosomes_keep_the_whole_genome_band() { + let sites: Vec = (1..=22).collect(); + let r = roh_scope_summary(&sites, "RG\tS\tchr2\t10000001\t18000000\t8000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.003)); + assert_eq!(at(&r, &["relationship"]), &Value::Str("no sign of related parents".into())); + assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(2_658_001_522)); + assert_eq!(at(&r, &["scope", "partial"]), &Value::Bool(false)); +} + +#[test] +fn roh_scope_a_segment_only_file_never_establishes_scope() { + let r = roh_scope_summary(&[], "RG\tS\tchr20\t1000001\t22000000\t21000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.3515)); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + assert_eq!(at(&r, &["upd_flag"]), &Value::Null); + assert_eq!(at(&r, &["scope", "source"]), &Value::Str("segments_only".into())); + let empty = roh_scope_summary(&[], "# empty\n"); + assert_eq!(at(&empty, &["f_roh"]), &Value::Null); +} + +#[test] +fn roh_scope_upd_is_a_region_hint_without_syndrome_names() { + let sites: Vec = (1..=22).collect(); + // Distal chr15 and chr11 runs do not establish the imprinted locus's + // parental origin, but mixed iso/heterodisomy can involve that locus. + for (chrom, start, end, length, region) in [ + (15, 40000001, 101991189, 61991189, "15q11-q13 (SNRPN, UBE3A)"), + (11, 60000001, 100000000, 40000000, "11p15 (H19/IGF2, KCNQ1OT1)"), + (20, 1000001, 22000000, 21000000, "20q13.32 (GNAS)"), + ] { + let r = roh_scope_summary(&sites, &format!("RG\tS\tchr{chrom}\t{start}\t{end}\t{length}\n")); + let u = at(&r, &["upd_flag"]); + assert_eq!(at(u, &["imprinted", "0", "region"]), &Value::Str(region.into())); + assert_eq!(at(u, &["imprinted", "0", "maternal_upd"]), &Value::Str("-".into())); + assert_eq!(at(u, &["imprinted", "0", "paternal_upd"]), &Value::Str("-".into())); + let note = text(at(u, &["note"])); + assert!(note.contains("deletion") && note.contains("acquired") && note.contains("parent")); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + } +} From 2fbbc528274fc8b293faafcb93b1cd00790a6888 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:04:02 +0100 Subject: [PATCH 34/86] test(setup): require clinical data in a default setup Bead: D11 --- scripts/ci/fake-docker/lib.sh | 23 ++++++++++ tests/fake-docker/base-setup.sh | 4 +- tests/fake-docker/setup-clinical-default.sh | 47 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/fake-docker/setup-clinical-default.sh diff --git a/scripts/ci/fake-docker/lib.sh b/scripts/ci/fake-docker/lib.sh index d8a28d811..27b6069dd 100644 --- a/scripts/ci/fake-docker/lib.sh +++ b/scripts/ci/fake-docker/lib.sh @@ -28,6 +28,29 @@ REF_FASTA_MD5=$(fake_body "$REF_FASTA_URL" | fake_md5) REF_FAI_MD5=$(fake_body "${REF_FASTA_URL}.fai" | fake_md5) export REF_FASTA_MD5 REF_FAI_MD5 +# Clinical setup uses real archive layouts, served by the existing fake curl +# and wget. CHECKSUMS is shared by basename, so it lists both VEP archives. +export FAKE_DOWNLOAD_DIR="${CASE_WORK}/downloads" +mkdir -p "$FAKE_DOWNLOAD_DIR" +for rel in 116 115; do + mkdir -p "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38" + printf 'fake VEP cache\n' > "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38/info.txt" + tar -czf "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_${rel}_GRCh38.tar.gz" -C "${CASE_WORK}/cache" "homo_sapiens/${rel}_GRCh38" +done +for rel in 116 115; do + name="homo_sapiens_vep_${rel}_GRCh38.tar.gz" + printf '%s %s\n' "$(sum -r "${FAKE_DOWNLOAD_DIR}/${name}" | awk '{print $1, $2}')" "$name" +done > "${FAKE_DOWNLOAD_DIR}/CHECKSUMS" +mkdir -p "${CASE_WORK}/bundle/data/grch38" +printf 'fake PCGR data\n' > "${CASE_WORK}/bundle/data/grch38/README" +tar -czf "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/bundle" data +if command -v sha256sum >/dev/null 2>&1; then + PCGR_DATA_SHA256=$(sha256sum "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') +else + PCGR_DATA_SHA256=$(shasum -a 256 "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') +fi +export PCGR_DATA_SHA256 + fail() { echo "ASSERT FAIL: $*" >&2 exit 1 diff --git a/tests/fake-docker/base-setup.sh b/tests/fake-docker/base-setup.sh index d373d9df9..6e5412647 100755 --- a/tests/fake-docker/base-setup.sh +++ b/tests/fake-docker/base-setup.sh @@ -13,7 +13,9 @@ output_lacks setup 'unbound variable' output_has setup 'Setup complete!' for f in reference/GRCh38_no_alt_analysis_set.fasta reference/GRCh38_no_alt_analysis_set.fasta.fai \ - clinvar/clinvar.vcf.gz clinvar/clinvar.vcf.gz.tbi; do + clinvar/clinvar.vcf.gz clinvar/clinvar.vcf.gz.tbi \ + vep_cache/homo_sapiens/116_GRCh38/info.txt vep_cache/homo_sapiens/115_GRCh38/info.txt \ + pcgr_data/20260620/data/grch38/README; do [ -s "${G}/${f}" ] || fail "setup.sh did not download ${f}" done grep -q '^image inspect' "$FAKE_DOCKER_LOG" || fail "setup.sh never checked a docker image" diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh new file mode 100644 index 000000000..4c289cdde --- /dev/null +++ b/tests/fake-docker/setup-clinical-default.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Default clinical downloads, explicit opt-out, disk gates and corrupt input. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" +export FETCH_TRIES=1 FETCH_WAIT=0 +G="${CASE_WORK}/without" +run_expect 0 without "${SCRIPTS}/setup.sh" --no-clinical-data "$G" +output_has without 'Setup complete!' +if grep -E '^(wget|curl).*homo_sapiens_vep_|^(wget|curl).*pcgr_ref_data' "$FAKE_DOCKER_LOG"; then + fail '--no-clinical-data downloaded clinical data' +fi + +G="${CASE_WORK}/clinical" +run_expect 0 clinical "${SCRIPTS}/setup.sh" --clinical-data "$G" +for rel in 116 115; do + [ -s "${G}/vep_cache/homo_sapiens/${rel}_GRCh38/info.txt" ] || fail "VEP ${rel} missing" +done +[ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'PCGR bundle missing' +output_lacks clinical 'Phase 1:|Docker Images' +# A second install does no network work, even with too little space. +cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" +run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" +cmp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" || fail 'installed clinical data downloaded again' + +mkdir -p "${CASE_WORK}/low-bin" +cat > "${CASE_WORK}/low-bin/df" <<'DF' +#!/usr/bin/env bash +echo 'Filesystem 1024-blocks Used Available Capacity Mounted on' +echo 'fakefs 41943040 0 41943040 0% /' +DF +chmod +x "${CASE_WORK}/low-bin/df" +G="${CASE_WORK}/low" +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 0 low "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_has low '56 GB.*5 GB margin' +output_has low '52 GB.*5 GB margin' +[ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'low-space VEP installed' +[ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'small bundle skipped with 40 GB free' + +# The digest belongs to the original fixture. Serve a different valid tarball. +printf 'corrupt\n' >> "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" +G="${CASE_WORK}/corrupt" +run_expect 0 corrupt "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_has corrupt 'sha256 checksum.*expected' +output_has corrupt '\[WARN\].*PCGR' +[ ! -e "${G}/pcgr_data/20260620" ] || fail 'corrupt bundle published' +[ ! -e "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz.part" ] || fail 'bad bundle kept' +[ -s "${G}/vep_cache/homo_sapiens/115_GRCh38/info.txt" ] || fail 'bundle failure stopped remaining install' From b81d49a420c98f236745dcd9998f6a8fb1745ebe Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:04:47 +0100 Subject: [PATCH 35/86] test(pgx): pin the disease-gene drug warnings for carriers, SCN1A and provisional levels Red on main: a single heterozygous urea-cycle hit warns, SCN1A warns, the level reads as a settled CPIC level. Bead: pgx-21 Bead: pgx-22 Bead: pgx-23 Bead: D13 --- .../og-pgx/tests/i411_contraindications.rs | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 rust/crates/og-pgx/tests/i411_contraindications.rs diff --git a/rust/crates/og-pgx/tests/i411_contraindications.rs b/rust/crates/og-pgx/tests/i411_contraindications.rs new file mode 100644 index 000000000..33ce246ba --- /dev/null +++ b/rust/crates/og-pgx/tests/i411_contraindications.rs @@ -0,0 +1,140 @@ +//! The disease-gene drug warnings of `og pgx cpic-report` against the shipped +//! `assets/pgx_contraindications.tsv` (issue #411, pgx-21, pgx-22, pgx-23): +//! +//! - one heterozygous pathogenic variant in a recessive urea-cycle gene +//! (ASL, ASS1, CPS1, NAGS) makes a carrier: a NOTE, never a WARNING; +//! - both copies changed, or two variants in the gene, give the WARNING; +//! - one POLG variant still warns (a known valproate risk); +//! - SCN1A gives nothing: no label or guideline names it; +//! - the CPIC level is printed as the provisional level it is. + +use std::collections::HashSet; + +use og_pgx::sections::contraindication_lines; +use og_pgx::tables::{ClinvarHit, Contraindication, read_contraindications}; + +fn table() -> Vec { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../assets/pgx_contraindications.tsv" + ); + read_contraindications(path).unwrap() +} + +fn hit(gene: &str, pos: &str, alt: &str, genotype: &str) -> ClinvarHit { + ClinvarHit { + chrom: "chr13".to_owned(), + pos: pos.to_owned(), + ref_: "G".to_owned(), + alt: alt.to_owned(), + genotype: genotype.to_owned(), + geneinfo: Some(format!("{gene}:1")), + clnsig: Some("Pathogenic".to_owned()), + } +} + +fn lines(hits: &[ClinvarHit]) -> Vec { + let meds: HashSet = ["valproate".to_owned()].into(); + contraindication_lines(hits, &table(), &meds) +} + +fn warnings(out: &[String]) -> Vec<&str> { + out.iter() + .map(|l| l.trim()) + .filter(|l| l.starts_with("WARNING:")) + .collect() +} + +#[test] +fn one_heterozygous_urea_cycle_hit_is_a_note() { + for gene in ["ASL", "ASS1", "CPS1", "NAGS"] { + let out = lines(&[hit(gene, "40000000", "A", "0/1")]); + assert!(warnings(&out).is_empty(), "{gene}: {out:#?}"); + let note = format!( + " NOTE: valproic acid -- {gene} chr13:40000000 G>A, genotype 0/1, ClinVar Pathogenic; ON THE MEDICATION LIST." + ); + assert_eq!(out.first(), Some(¬e), "{gene}: {out:#?}"); + assert!( + out.iter().any(|l| l.contains("a carrier, not a patient")), + "{gene}: {out:#?}" + ); + } + // A phased or partly missing genotype is still one changed copy. + for gt in ["0|1", "1|0", "./1"] { + let out = lines(&[hit("ASS1", "40000000", "A", gt)]); + assert!(warnings(&out).is_empty(), "{gt}: {out:#?}"); + } +} + +#[test] +fn both_copies_or_two_variants_in_a_urea_cycle_gene_warn() { + for gt in ["1/1", "1|1", "1"] { + let out = lines(&[hit("ASS1", "40000000", "A", gt)]); + assert_eq!(warnings(&out).len(), 1, "{gt}: {out:#?}"); + assert!( + warnings(&out)[0].contains("both copies changed"), + "{gt}: {out:#?}" + ); + } + // Two variants in trans, at two positions or as the two alleles of one + // split multi-allelic site: one WARNING for each. + for (pos2, alt2) in [("40000500", "A"), ("40000000", "C")] { + let out = lines(&[ + hit("CPS1", "40000000", "A", "0/1"), + hit("CPS1", pos2, alt2, "1/0"), + ]); + let w = warnings(&out); + assert_eq!(w.len(), 2, "{pos2}>{alt2}: {out:#?}"); + assert!(w.iter().all(|l| l.contains("2 variants in CPS1")), "{w:#?}"); + assert!(!out.iter().any(|l| l.contains("NOTE:")), "{out:#?}"); + } + // The same variant listed twice (two ClinVar records) is one variant. + let out = lines(&[ + hit("NAGS", "40000000", "A", "0/1"), + hit("NAGS", "40000000", "A", "0/1"), + ]); + assert!(warnings(&out).is_empty(), "{out:#?}"); +} + +#[test] +fn one_polg_or_otc_variant_still_warns() { + for gene in ["POLG", "OTC"] { + let out = lines(&[hit(gene, "89300000", "T", "0/1")]); + assert_eq!( + warnings(&out), + [format!( + "WARNING: valproic acid -- {gene} chr13:89300000 G>T, genotype 0/1, ClinVar Pathogenic; ON THE MEDICATION LIST." + )], + "{out:#?}" + ); + } +} + +#[test] +fn an_scn1a_hit_gives_no_drug_line() { + assert!(!table().iter().any(|r| r.gene == "SCN1A")); + let out = lines(&[hit("SCN1A", "166000000", "T", "0/1")]); + assert!(out.is_empty(), "{out:#?}"); +} + +#[test] +fn every_row_quotes_a_label() { + let bad: Vec = table() + .into_iter() + .filter(|r| !r.label_source.starts_with("https://dailymed.nlm.nih.gov/")) + .map(|r| format!("{} {}", r.gene, r.drug)) + .collect(); + assert!(bad.is_empty(), "rows no label names: {bad:?}"); +} + +#[test] +fn the_cpic_level_reads_as_provisional() { + let out = lines(&[hit("POLG", "89300000", "T", "0/1")]); + assert!( + out.iter().any(|l| l.ends_with( + "(provisional CPIC level A/B: no CPIC guideline covers this pair)" + )), + "{out:#?}" + ); + assert!(!out.iter().any(|l| l.ends_with("(CPIC level A/B)")), "{out:#?}"); +} From 594270158406e0f1f67fc07bcb8d19ae25a06f02 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:05:01 +0100 Subject: [PATCH 36/86] test(live): check inline reads and same-origin viewer framing Bead: pgp-iqon.11 --- tests/test_live_viewer.py | 127 ++++++++++++++++++++++++++++++++++- tests/test_owngenome_live.py | 1 + 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 2521c816d..afe6ba730 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -33,6 +33,8 @@ Run: python3 tests/test_live_viewer.py """ import http.client +import gzip +import json import importlib.util import os import re @@ -51,6 +53,7 @@ import owngenome_live # noqa: E402 import render_report # noqa: E402 import test_viewer as tv # noqa: E402 +import test_owngenome_live as tl # noqa: E402 PASSWORD = "viewer test password" FAILED = [] @@ -113,7 +116,7 @@ def viewer_checks(port, auth, s, label=""): code, h, body = request(port, "/runs/S/viewer/", auth) csp = h.get("content-security-policy", "") check(f"{label}page: 200 with the viewer's policy, served by the site", - code == 200 and csp == tv.serve.CSP and b"served by the run site" in body + code == 200 and csp == tv.serve.CSP.replace("frame-ancestors 'none'", "frame-ancestors 'self'") and b"served by the run site" in body and b"served from this computer only" not in body, (code, csp, body[:300])) code, _, body = request(port, "/runs/S/viewer/igv.min.js", auth) c2, _, js = request(port, "/runs/S/viewer/viewer.js", auth) @@ -170,6 +173,126 @@ def outside_checks(port, auth, label=""): url == "data/0/my%20ref%231%3F.fasta" and code == 206 and body == b">chr1", (url, code, body)) +def framing_checks(port): + for path in ("/runs/S/viewer/", "/runs/S/viewer/viewer.js", "/runs/S/viewer/config.json", + "/runs/S/viewer/data/2/S_sorted.bam"): + code, h, _ = request(port, path) + csp = h.get("content-security-policy", "") + check("same-origin viewer framing: " + path, + code == 200 and "frame-ancestors 'self'" in csp + and h.get("x-frame-options") == "SAMEORIGIN", (code, h)) + for path in ("/", "/login", "/assets/map.js", "/runs/S/", "/runs/S/state.json", + "/runs/S/report.html", "/not-found"): + _, h, _ = request(port, path) + check("non-viewer framing stays closed: " + path, + "frame-ancestors 'none'" in h.get("content-security-policy", "") + and h.get("x-frame-options") == "DENY", h) + + +def map_reads_checks(srv, work, chrome): + """The real map and viewer, on invented map rows and test_viewer's synthetic reads. + + No product data loader is bypassed: the fixture endpoint supplies the same + gzip document as map.json. The page supplies its existing veil callbacks. + """ + app, port = srv[0].app, srv[1].server_address[1] + policy = {"block": False} + page = ('' + '' + '' + '') + doc = {"v": 1, "people": [{"id": "S"}], "dict": {"veil": ["closed"]}, + "ref": {"chroms": [{"n": "chr1", "len": 60000, "masked": []}], "bands": {}}, + "per": {"S": {"findings": { + "id": ["open", "closed"], "level": ["review", "act"], "gate": ["", ""], + "chrom": ["chr1", "chr1"], "pos": [451, 551], "s": [450, 550], "e": [451, 551], + "gene": ["", ""], "title": ["Synthetic finding", "Closed finding"], + "href": ["#open", "#closed"], "phrase": ["Synthetic row", "Closed row"], "veil": [-1, 0]}}}, + "layers": {"findings": {"state": "landed"}, "roh": {"state": "not_run"}}, "text": {}} + data = gzip.compress(json.dumps(doc).encode()) + + def fixture(req, _): + csp = owngenome_live.CSP + if policy["block"]: + csp = csp.replace("frame-src 'self'; ", "") + return req.send_body(200, "text/html", page, csp=csp) + + app.route("GET", r"/map-1g/", fixture) + app.route("GET", r"/map-1g.json", lambda req, _: req.send_body(200, "application/gzip", data)) + b = tl.Chrome(chrome, f"{work}/chrome-map-1g") + try: + b.call("Emulation.setEmulatedMedia", {"features": [{"name": "prefers-color-scheme", "value": "dark"}, + {"name": "prefers-reduced-motion", "value": "reduce"}]}, b.session) + + def mount(demo=False): + b.go(f"http://127.0.0.1:{port}/map-1g/#chr1:1-49000") + if not b.wait("window.PGPMap && window.PGPUI"): + return False + b.eval("window.closedResult = true; " + "document.documentElement.dataset.mode = " + json.dumps("replay" if demo else "live") + "; " + "window.testMap = PGPMap.create({url:'/map-1g.json'," + "veil:function(id){return id==='closed' && window.closedResult ? 'closed':'open';}," + "known:function(){return true;},now:function(){return true;}});" + "document.body.appendChild(testMap.node);testMap.refresh('one');") + return b.wait("document.querySelector('.gmap[data-level=reads]')") + + def loaded(): + return b.wait("(function(){var f=document.querySelector('.gm-reads iframe');" + "return f && f.contentDocument && f.contentDocument.documentElement.dataset.pgpStatus==='ready';})()", 30) + + check("map: opens at a 49 kb view", mount()) + check("map: under 50 kb offers the reads pill", b.eval("!!document.querySelector('.gm-reads-pill:not([hidden])')")) + if not b.eval("!!document.querySelector('.gm-reads-pill')"): + return + b.eval("document.querySelector('.gm-reads-pill').click()") + check("map: the reads panel loads the viewer in an iframe at the nearest open finding", loaded(), + b.eval("document.querySelector('.gm-reads').textContent")) + check("map: the iframe locus is 1-based, finding plus or minus 50", + b.eval("document.querySelector('.gm-reads iframe').contentWindow.location.hash") == "#chr1:401-501") + check("map: a closed result has no card or reads link", + b.eval("!document.querySelector('.gm-marks [data-id=closed]') && !document.querySelector('.gm-card-reads[data-id=closed]')")) + # A moved view must update the existing frame, with no page reload. + b.eval("window.savedFrame=document.querySelector('.gm-reads iframe');location.hash='chr1:1-49001'") + check("map: moving updates the same iframe to the map's locus", + b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) + check("viewer: alignments remain visible at 49 kb", + b.wait("(function(){var d=savedFrame.contentDocument;return d.querySelector('canvas.igv-canvas') && " + "!Array.from(d.querySelectorAll('.igv-zoom-in-notice-container')).some(function(x){return x.offsetHeight>0;});})()")) + b.eval("document.querySelector('.gm-reads-close').dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") + check("map: Escape closes the reads and returns focus without moving the map", + b.eval("document.querySelector('.gm-reads').hidden && document.activeElement===document.querySelector('.gm-canvas') && " + "document.querySelector('.gmap').dataset.view==='chr1:1-49001'")) + b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown',{key:'r',bubbles:true}))") + check("map: R reopens the panel", loaded()) + b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('[data-stop=reads]').click()") + check("map: Reads gauge opens a 2 kb view and the panel", b.wait("!document.querySelector('.gm-reads').hidden && " + "document.querySelector('.gmap').dataset.level==='reads'")) + b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('.gm-marks [data-id=open]').focus()") + b.eval("document.querySelector('.gm-card-reads').click()") + check("map: the card opens reads at its finding", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:401-501'")) + b.eval("window.closedResult=false;testMap.refresh('one');document.querySelector('.gm-marks [data-id=closed]').focus();" + "document.querySelector('.gm-card-reads').click()") + check("map: an opened finding offers reads", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:501-601'")) + b.eval("window.closedResult=true;testMap.refresh('one')") + check("map: closing a result removes its open reads panel and link", + b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads')")) + + # CI proves the browser assertion detects the original policy mistake. + policy["block"] = True + mount() + b.eval("window.blockedFrames=0;document.addEventListener('securitypolicyviolation',function(e){" + "if(e.violatedDirective==='frame-src')window.blockedFrames++;});document.querySelector('.gm-reads-pill').click()") + check("control: without frame-src the reads iframe is blocked", b.wait("window.blockedFrames>0") and not loaded()) + policy["block"] = False + mount(demo=True) + b.eval("document.querySelector('.gm-reads-pill').click()") + check("map: the static demo explains why it has no viewer", + b.wait("document.querySelector('.gm-reads').textContent.includes('The genome viewer is not in the demo')") + and b.eval("!document.querySelector('.gm-reads iframe')")) + finally: + b.close() + + def mutant_dir(work, name, old, new): """A plugin folder holding bin/owngenome_live_viewer.py with one planted mistake (beside a link to assets/, which the plugin finds from its own folder).""" @@ -269,6 +392,7 @@ def main(): try: p = srv[1].server_address[1] outside_checks(p, {}) + framing_checks(p) # 7. headless Chrome, on the site without a password (loopback) chrome = tv.find_chrome() @@ -290,6 +414,7 @@ def main(): status and status.group(1) == "ready" and tracks and tracks.group(1) == "3", (status and status.group(1), r.stderr[-800:], r.stdout[:600])) check("browser: the policy blocked nothing the page needed", not blocked, blocked and blocked.group(1)) + map_reads_checks(srv, work, chrome) finally: stop(*srv) diff --git a/tests/test_owngenome_live.py b/tests/test_owngenome_live.py index 29f84d7fa..1296d5b5e 100644 --- a/tests/test_owngenome_live.py +++ b/tests/test_owngenome_live.py @@ -841,6 +841,7 @@ def stored_leaks(text): code, _, _ = request(port, "GET", "/nothing/here", headers=auth) bad = [p for p, hdr in RESPONSES if "default-src 'none'" not in hdr.get("content-security-policy", "") or "frame-ancestors 'none'" not in hdr.get("content-security-policy", "") + or hdr.get("x-frame-options") != "DENY" or hdr.get("x-content-type-options") != "nosniff" or hdr.get("cache-control") != "no-store"] check(f"every browser response ({len(RESPONSES)}) carries the policy, nosniff and no-store", not bad, bad) texts = {f: open(os.path.join(ASSETS, f)).read() for f in os.listdir(ASSETS) From cd57a630d32e2e1bb0a643f99b5e0f34d1f17ee6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:05:59 +0100 Subject: [PATCH 37/86] test(freshness): require metadata checks for clinical downloads Bead: D11 --- .../tests/screen_setup_clinical_default.rs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 rust/crates/og-fresh/tests/screen_setup_clinical_default.rs diff --git a/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs new file mode 100644 index 000000000..077cd2a27 --- /dev/null +++ b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs @@ -0,0 +1,53 @@ +//! Clinical links must be live and metadata must agree, without archive GETs. +use og_fresh::http::{Body, Fetch, GitHub, Headers, Request, Want}; +use og_fresh::lookups::{Ctx, DATABASES}; +use og_fresh::pins::Pins; +use og_fresh::selftest::FakeRegistry; +use og_fresh::{Lookup, failed}; +use std::path::Path; +use std::sync::Mutex; + +struct Downloads { + github: GitHub, + missing: bool, + bad_length: bool, + bad_checksum: bool, + calls: Mutex>, +} +impl Fetch for Downloads { + fn github_api(&self) -> &GitHub { &self.github } + fn request(&self, r: &Request<'_>) -> Lookup<(Body, Headers)> { + self.calls.lock().unwrap().push((r.method.into(), r.url.into())); + if self.missing { return failed("HTTP 404: release does not exist"); } + if r.url.ends_with("CHECKSUMS") { + assert_eq!(r.want, Want::Text); + let body = if self.bad_checksum { "123 5 wrong.tar.gz\n".into() } else { + "123 5 homo_sapiens_vep_116_GRCh38.tar.gz\n456 6 homo_sapiens_vep_115_GRCh38.tar.gz\n".into() + }; + Ok((Body::Text(body), Headers::default())) + } else { + assert_eq!(r.method, "HEAD", "large archive body was requested"); + Ok((Body::None, Headers::from_pairs(vec![("Content-Length".into(), if self.bad_length { "1" } else { "7664082467" }.into())]))) + } + } +} +fn check(missing: bool, bad_length: bool, bad_checksum: bool) { + let fetch = Downloads { github: GitHub { api: "https://invalid".into(), token: None }, missing, bad_length, bad_checksum, calls: Mutex::new(vec![]) }; + let registry = FakeRegistry { tags: |_| panic!("registry is not needed"), digests: |_| panic!("registry is not needed"), asked: Mutex::new(vec![]) }; + let pins = Pins::parse("VEP_CACHE_RELEASE=\"116\"\nPCGR_VEP_CACHE_RELEASE=\"115\"\nPCGR_DATA_BUNDLE=\"20260620\"\nPCGR_DATA_BYTES=\"7664082467\"\n"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let common = root.join("scripts/lib/common.sh"); + let c = Ctx { fetch: &fetch, registry: ®istry, pins: &pins, versions: Path::new("unused"), setup: Path::new("unused"), common: &common, now: "2026-11-15T00:00:00Z".parse().unwrap() }; + for name in ["VEP cache download", "CPSR cache download", "PCGR bundle download"] { + let (_, lookup) = DATABASES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("missing liveness row: {name}")); + let got = lookup(&c); + let should_fail = missing || (bad_length && name == "PCGR bundle download") || (bad_checksum && name != "PCGR bundle download"); + assert_eq!(got.is_err(), should_fail, "{name}: {got:?}"); + } + let calls = fetch.calls.lock().unwrap(); + assert!(calls.iter().any(|(_, url)| url.contains("release-115/"))); +} +#[test] fn clinical_links_head_only() { check(false, false, false); } +#[test] fn missing_release_fails() { check(true, false, false); } +#[test] fn changed_bundle_length_fails() { check(false, true, false); } +#[test] fn missing_cache_checksum_fails() { check(false, false, true); } From 73175279dddc5b89b28d45560d0635720f8cf1de Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:06:28 +0100 Subject: [PATCH 38/86] test(bloodtype): pin the record a marker reads when several overlap it Four cases that fail on main, each read through the index and by a full pass: an ABO c.261 insertion after an all-sites reference record or a split SNV record, a later no-call record of the marker allele, a shorter reference record at the A2 deletion marker, and a padded record that spans KEL c.578 with another base there. Bead: trait-40 Bead: trait-41 Bead: trait-29 --- rust/crates/og-bloodtype/tests/report.rs | 208 +++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index c025c3e7b..893169aa8 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -215,3 +215,211 @@ fn an_index_older_than_its_vcf_is_not_used() { let (_, how) = read_vcf(&gz.to_string_lossy(), read_markers(&markers()).unwrap()).unwrap(); assert_eq!(how, Access::Stream); } + +/// A one-sample VCF record line. +fn line(chrom: &str, pos: i64, r: &str, alt: &str, filt: &str, gt: &str) -> String { + format!("{chrom}\t{pos}\t.\t{r}\t{alt}\t50\t{filt}\t.\tGT\t{gt}\n") +} + +/// `records` (sorted, under 64 KiB) as a plain VCF and as a bgzipped VCF +/// with a `.tbi`: one bin per sequence whose one chunk spans that +/// sequence's lines, the smallest index the reader takes. +fn both_ways(d: &Path, records: &[String]) -> (PathBuf, PathBuf) { + use std::io::Write; + let text = format!("{HEAD}{}", records.concat()); + let plain = d.join("s.vcf"); + std::fs::write(&plain, &text).unwrap(); + let gz = d.join("s.vcf.gz"); + let mut w = og_io::bgzf::Writer::new( + std::fs::File::create(&gz).unwrap(), + std::num::NonZero::new(1).unwrap(), + ) + .unwrap(); + w.write_all(text.as_bytes()).unwrap(); + w.finish().unwrap(); + // Each sequence's (first byte, end byte) in the one block's text. + let mut seqs: Vec<(String, u64, u64)> = Vec::new(); + let mut at = HEAD.len() as u64; + for r in records { + let chrom = r.split('\t').next().unwrap().to_owned(); + let end = at + r.len() as u64; + match seqs.last_mut() { + Some(s) if s.0 == chrom => s.2 = end, + _ => seqs.push((chrom, at, end)), + } + at = end; + } + let names: Vec = seqs + .iter() + .flat_map(|s| s.0.bytes().chain([0])) + .collect(); + let mut tbi: Vec = b"TBI\x01".to_vec(); + let i32s = |v: &mut Vec, xs: &[i32]| xs.iter().for_each(|x| v.extend(x.to_le_bytes())); + // n_ref, VCF preset, columns 1 2 0, meta '#', skip 0, names. + i32s(&mut tbi, &[seqs.len() as i32, 2, 1, 2, 0, b'#' as i32, 0]); + i32s(&mut tbi, &[names.len() as i32]); + tbi.extend(&names); + for (_, beg, end) in &seqs { + // One bin (0, the whole sequence), one chunk, no linear index. + i32s(&mut tbi, &[1, 0, 1]); + tbi.extend(beg.to_le_bytes()); + tbi.extend(end.to_le_bytes()); + i32s(&mut tbi, &[0]); + } + std::fs::write(d.join("s.vcf.gz.tbi"), tbi).unwrap(); + (plain, gz) +} + +/// (status, alt copies, note) of the site `name`, read through the index +/// and by a full pass, which must agree. +fn site_both_ways(d: &Path, records: &[String], name: &str) -> (String, Option, String) { + let (plain, gz) = both_ways(d, records); + let (whole, how) = + read_vcf(&plain.to_string_lossy(), read_markers(&markers()).unwrap()).unwrap(); + assert_eq!(how, Access::Stream); + let (indexed, how) = + read_vcf(&gz.to_string_lossy(), read_markers(&markers()).unwrap()).unwrap(); + assert_eq!(how, Access::Index); + assert_eq!(indexed, whole, "{records:?}"); + let s = &whole[name]; + (s.status.as_str().to_owned(), s.n, s.note.clone()) +} + +#[test] +fn the_abo_insertion_counts_after_a_reference_or_other_record() { + let d = dir("abo261"); + let ins = |gt: &str| line("chr9", 133257521, "T", "TC", "PASS", gt); + for (first, gt, copies) in [ + // An all-sites VCF (bcftools call -m without -v): the reference + // record comes first, then the insertion. + (line("chr9", 133257521, "T", ".", ".", "0/0"), "0/1", 1), + (line("chr9", 133257521, "T", ".", ".", "0/0"), "1/1", 2), + // A gVCF-style reference record. + (line("chr9", 133257521, "T", "<*>", ".", "0/0"), "0/1", 1), + ] { + for recs in [[first.clone(), ins(gt)], [ins(gt), first.clone()]] { + let (status, n, note) = site_both_ways(&d, &recs, "ABO_261"); + assert_eq!( + (status.as_str(), n, note.as_str()), + ("called", Some(copies), ""), + "{recs:?}" + ); + } + } + // A split multi-allelic record (bcftools norm -m-any writes the SNV + // first): the insertion counts and the SNV is named. + let snv = line("chr9", 133257521, "T", "G", "PASS", "1/0"); + for recs in [[snv.clone(), ins("0/1")], [ins("0/1"), snv.clone()]] { + let (status, n, note) = site_both_ways(&d, &recs, "ABO_261"); + assert_eq!((status.as_str(), n), ("called", Some(1)), "{recs:?}"); + assert_eq!(note, "another allele at this position (T>G, genotype 1/0)"); + } + // The whole report: A1/O.01, not O. + let t = run( + &d, + &[line("chr9", 133257521, "T", ".", ".", "0/0"), ins("0/1")], + Some(depth(&d, 2.0, &HashMap::new())), + ); + assert_eq!(t["ABO"][..3], ["A1", "A1/O.01", "high"]); +} + +#[test] +fn a_later_record_of_the_marker_allele_that_is_no_call_wins_over_a_reference_record() { + let d = dir("abo261-nocall"); + let r = line("chr9", 133257521, "T", ".", ".", "0/0"); + for (later, note) in [ + (line("chr9", 133257521, "T", "TC", "PASS", "./."), "genotype ./."), + ( + line("chr9", 133257521, "T", "TC", "LowQual", "0/1"), + "FILTER LowQual", + ), + ] { + for recs in [[r.clone(), later.clone()], [later.clone(), r.clone()]] { + let (status, n, got) = site_both_ways(&d, &recs, "ABO_261"); + assert_eq!( + (status.as_str(), n, got.as_str()), + ("no call", None, note), + "{recs:?}" + ); + } + } +} + +#[test] +fn a_shorter_reference_record_leaves_a_deletion_marker_to_the_reads() { + let d = dir("abo1061"); + let ins = line("chr9", 133257521, "T", "TC", "PASS", "0/1"); + // An all-sites VCF writes `C . 0/0` at the A2 marker (REF CG): it says + // nothing past its own base, so the reads decide, as with no record. + for r in [ + line("chr9", 133255669, "C", ".", ".", "0/0"), + line("chr9", 133255669, "C", "<*>", ".", "0/0"), + ] { + let recs = [r.clone(), ins.clone()]; + assert_eq!( + site_both_ways(&d, &recs, "ABO_1061"), + ("absent".to_owned(), None, String::new()) + ); + let t = run(&d, &recs, Some(depth(&d, 2.0, &HashMap::new()))); + assert_eq!(t["ABO"][..3], ["A1", "A1/O.01", "high"], "{r}"); + // The deletion after it still counts, in either order. + let del = line("chr9", 133255669, "CG", "C", "PASS", "0/1"); + for recs in [[r.clone(), del.clone()], [del.clone(), r.clone()]] { + let (status, n, _) = site_both_ways(&d, &recs, "ABO_1061"); + assert_eq!((status.as_str(), n), ("called", Some(1)), "{recs:?}"); + } + } + // A shorter record with a non-reference call stays a no call. + let (status, _, note) = site_both_ways( + &d, + &[line("chr9", 133255669, "C", "T", "PASS", "0/1")], + "ABO_1061", + ); + assert_eq!( + (status.as_str(), note.as_str()), + ("no call", "REF C at this position, the marker's REF is CG") + ); +} + +#[test] +fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { + let d = dir("kel578"); + let reads = || Some(depth(&d, 2.0, &HashMap::new())); + // KEL c.578C>T (K) is chr7:142957921 G>A. A padded record one base + // upstream carries the same change (CG>CA), or a deletion of that + // base with it (DeepVariant's CG>C,CA 1/2). + for (r, alt, gt) in [("CG", "CA", "1/1"), ("CG", "C,CA", "1/2"), ("CG", "C", "0/1")] { + let recs = [line("chr7", 142957920, r, alt, "PASS", gt)]; + let (status, n, note) = site_both_ways(&d, &recs, "KEL_578"); + assert_eq!( + (status.as_str(), n, note.as_str()), + ( + "no call", + None, + format!("a record at chr7:142957920 ({r}>{alt}, genotype {gt}) spans this site") + .as_str() + ) + ); + let t = run(&d, &recs, reads()); + assert_eq!(t["Kell"][0], "K/k not determined", "{recs:?}"); + assert_ne!(t["Kell"][2], "high", "{recs:?}"); + } + // Split (bcftools norm -m-any): the deletion before the SNV record + // still makes the SNV's copy count unknown (one copy may be deleted). + let recs = [ + line("chr7", 142957920, "CG", "C", "PASS", "0/1"), + line("chr7", 142957921, "G", "A", "PASS", "0/1"), + ]; + let (status, ..) = site_both_ways(&d, &recs, "KEL_578"); + assert_eq!(status, "no call"); + // A spanning record that keeps the marker's base, or calls reference, + // leaves the site to its own records and the reads. + for recs in [ + [line("chr7", 142957920, "CG", "TG", "PASS", "1/1")], + [line("chr7", 142957920, "CG", "C", "PASS", "0/0")], + [line("chr7", 142957920, "CG", "C", "PASS", "./.")], + ] { + let (status, ..) = site_both_ways(&d, &recs, "KEL_578"); + assert_eq!(status, "absent", "{recs:?}"); + } +} From 5fc18ba0b281c431dfb61ec6e941d7d127d831eb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:04 +0100 Subject: [PATCH 39/86] docs(licence): say GPL-3.0-or-later wherever OwnGenome states its licence CONTRIBUTING.md granted contributions under GPL-3.0 only; CLAUDE.md, the traits page and the Nextflow page said GPL-3.0. README, docs/index.md and rust/Cargo.toml already said GPL-3.0-or-later. Now every statement says GPL-3.0-or-later, README carries the GPL's or-later notice and the nextflow.config manifest names the licence. No DCO and no SPDX headers. Bead: lic-01 Bead: plat-75 Bead: lic-04 Bead: lic-05 Bead: D7 --- CLAUDE.md | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- docs/42-traits.md | 4 ++-- docs/nextflow.md | 2 +- nextflow.config | 1 + 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1b8788d1a..c713c6cfb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,7 +148,7 @@ Change its line in `versions.env`, plus the coupled data variable its comment na - All processing is local; genomic data never leaves the machine - Pin tool versions; never use floating tags - Reference genome: GRCh38, NCBI's no-ALT analysis set (`reference/GRCh38_no_alt_analysis_set.fasta`). No aligner here runs ALT-aware, so a reference with ALT contigs gives MAPQ 0 at CYP2D6, the MHC and KIR; `validate-setup.sh` refuses one (unless `ALLOW_ALT_REFERENCE=true`) and refuses a BAM whose @SQ lines differ from the `.fai`. A change of reference means realigning every sample (`docs/realignment.md`) -- License: GPL-3.0 +- License: GPL-3.0-or-later (README, CONTRIBUTING.md, `rust/Cargo.toml` and the `nextflow.config` manifest say the same) ## Tool-Specific Gotchas diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3382205e..a4929bcb3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,4 +123,4 @@ Be respectful. Genomic data is deeply personal. This project exists to empower i ## License -By contributing, you agree that your contributions will be licensed under GPL-3.0. +By contributing, you agree that your contributions will be licensed under GPL-3.0-or-later: the GNU General Public License, version 3 or (at your option) any later version, the same licence as the rest of the repository ([LICENSE](LICENSE), [README](README.md#license)). diff --git a/README.md b/README.md index e3465df5a..e2e53b432 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,4 @@ Your genome data is sensitive personal information. This pipeline runs locally a ## License -[GPL-3.0-or-later](LICENSE) +[GPL-3.0-or-later](LICENSE). OwnGenome is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Contributions come in under the same terms ([CONTRIBUTING.md](CONTRIBUTING.md#license)). diff --git a/docs/42-traits.md b/docs/42-traits.md index ad3aa25cf..8e85611bc 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -6,7 +6,7 @@ Reads your genotype at a small set of well-replicated variants and prints what e ## Why -These are the results most people find easiest to read, and the pipeline had no such section. SNPedia, the usual source, is CC BY-NC-SA, which a GPL-3.0 project cannot copy, so the table here is written from the primary papers instead: each row's reading follows the cited abstract or its open full text, and the alleles follow dbSNP's GRCh38 record. No text comes from SNPedia or any other database. +These are the results most people find easiest to read, and the pipeline had no such section. SNPedia, the usual source, is CC BY-NC-SA, which a GPL-3.0-or-later project cannot copy, so the table here is written from the primary papers instead: each row's reading follows the cited abstract or its open full text, and the alleles follow dbSNP's GRCh38 record. No text comes from SNPedia or any other database. ## Tool @@ -148,4 +148,4 @@ Seconds: bcftools reads the gVCF at the table's positions through its index (36 - Each traits table row is read alone. The genotype rules read several sites together: APOE, the TAS2R38 taster haplotype with read-backed phase, and the six rules of the table above. Two different MC1R red-hair variants on the same or on different copies are still read one by one; their note says so. - SNVs only; the table has no indel and no chrX, chrY or chrM row. - Positions with no genotype (no reads, or a no-call) are listed as not read. On the e2e fixture, a small slice of the HG002 genome, only the table positions inside its slices are read, and the e2e case checks their genotypes against the GIAB truth. -- The licence: the readings, notes and table are this project's own text, GPL-3.0 like the rest of the repository. +- The licence: the readings, notes and table are this project's own text, GPL-3.0-or-later like the rest of the repository. diff --git a/docs/nextflow.md b/docs/nextflow.md index 0f15b62c5..694f952aa 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -451,7 +451,7 @@ Cyrius (CYP2D6 star alleles) is under a non-commercial licence and is not in the ## Relationship to nf-core -This pipeline uses [nf-core](https://nf-co.re/) template patterns and tooling for code quality, but is **not an official nf-core pipeline** (it uses a GPL-3.0 license; nf-core requires MIT). +This pipeline uses [nf-core](https://nf-co.re/) template patterns and tooling for code quality, but is **not an official nf-core pipeline** (it uses the GPL-3.0-or-later license; nf-core requires MIT). Individual modules (PharmCAT, pypgx, slivar) will be contributed to [nf-core/modules](https://github.com/nf-core/modules) under MIT license for use by the broader community. diff --git a/nextflow.config b/nextflow.config index 46f83e521..e50156bf4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -271,6 +271,7 @@ manifest { homePage = 'https://github.com/GeiserX/OwnGenome' description = 'Whole genome sequencing from FASTQ, BAM or VCF to clinical interpretation' mainScript = 'main.nf' + license = 'GPL-3.0-or-later' // Validated on the Nextflow release pinned in versions.env (NEXTFLOW_VERSION) // and on the newest 26.10.x (both legs of .github/workflows/nextflow.yml). // The floor rejects 24.x, whose vcfanno strict-parser regression breaks this From da97715cad505a4ecea8fa4ed173293911cdf2df Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:04 +0100 Subject: [PATCH 40/86] docs(acmg): cite ACMG SF v3.3 and say the rule text is a short quotation docs/resources.md named ACMG SF v3.2 with 81 genes; it now names v3.3, 84 genes and the Lee 2025 DOI. The asset header and the clinical filter page say that table_text holds short cited quotations of a factual reporting rule and that "(See text)" points to the DCM section. A new og-summary test shows that # lines, at the top or between rows, do not change the table read from acmg_sf.tsv. Bead: lic-06 Bead: clin-43 Bead: D2 --- assets/acmg_sf.tsv | 7 +++-- docs/23-clinical-filter.md | 2 ++ docs/resources.md | 2 +- rust/crates/og-summary/src/collect/assets.rs | 32 ++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/assets/acmg_sf.tsv b/assets/acmg_sf.tsv index 9d7d27e6e..66c1fed5b 100644 --- a/assets/acmg_sf.tsv +++ b/assets/acmg_sf.tsv @@ -28,8 +28,11 @@ # pseudodeficiency allele is set aside; a hypomorph allele counts, but the finding says # partial, not profound. "." for none # sf_version the ACMG SF list version that added the gene (Table 1) -# table_text Table 1's "Variants to report" text -# note what Table 1's footnotes add, shown with the finding; "." +# table_text Table 1's "Variants to report" text, quoted as the paper writes it: short cited +# quotations of a factual reporting rule (six phrases, the longest 44 characters), with +# the citation above. "(See text)" points to the paper's section on the dilated +# cardiomyopathy (DCM) genes +# note what Table 1's footnotes add, in our own words, shown with the finding; "." gene hgnc_id inheritance report_if variants sv_cn sv_regions severity sf_version table_text note APC HGNC:583 AD any . no . . 1.0 All P and LP . RET HGNC:9967 AD any . no . . 1.0 All P and LP . diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index 66f03986c..c2b3088a5 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -88,6 +88,8 @@ The Nextflow `CLINICAL_FILTER` module applies the same tiers, ranking, frequency Both reports list, beside the clinical filter's counts, the secondary findings in the 84 genes of the ACMG SF v3.3 list (Lee et al., Genet Med 2025;27(8):101454; the genes in which the ACMG recommends reporting pathogenic variants found by chance, such as BRCA1, LDLR, MYBPC3 or TTN): the step 6 ClinVar hits in those genes, the clinical filter's records with a HIGH-impact consequence on the ranked transcript in them, and step 5's AnnotSV class 4 and 5 structural variants in the genes whose rule allows them. Each gene's rule from Table 1 of the list is a row of `assets/acmg_sf.tsv`: one variant in a dominant gene, two in a recessive one (MUTYH, BTD, ATP7B and others; a homozygote, or two variants of unknown phase), HFE only as homozygous p.C282Y, GLA and OTC also in a heterozygous female, ABCD1 hemizygous, homozygous or with two heterozygous variants. A variant the rule does not report is listed apart with the reason; a pseudodeficiency allele is not counted, and a BTD finding with the p.Asp444His hypomorph says partial, not profound. Of the reported HIGH-impact records the reports count as "rare HIGH impact" only the rare ones with a sound call; a weak call or an unknown frequency is listed with its mark and counted apart, and a common one is left out. The list version is `ACMG_SF_VERSION` in `bin/collect_summary.py`. It is a list to review with a clinician: a rare HIGH-impact variant is not a ClinVar classification. [Interpreting results](interpreting-results.md#carrier-status-is-not-disease) gives every rule. Step 17 (CPSR) reports the ACMG secondary findings with its own classification. +The `table_text` column of `assets/acmg_sf.tsv` quotes each gene's rule from Table 1 of the paper (doi:10.1016/j.gim.2025.101454), such as "All P and LP" or "P and LP (2 variants)". summary.json gives that text as the rule of each finding, and the reports use it in the reason a variant is not reported. These are short cited quotations of a factual reporting rule: six phrases, the longest 44 characters. "(See text)" after a rule points to the paper's section on the dilated cardiomyopathy (DCM) genes. The `note` column says what the table's footnotes add, in our own words. + ## Runtime 2.5 to 6.5 minutes, measured in three 30-40x WGS runs ([runtime](hardware-requirements.md#runtime-per-step)) (I/O-bound, reading the large VEP VCF) diff --git a/docs/resources.md b/docs/resources.md index e34492214..d90ddc14f 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -26,7 +26,7 @@ Free resources for understanding your genomic data, organized from most accessib ### Variant Interpretation - **[ClinGen](https://www.clinicalgenome.org/)** — The Clinical Genome Resource. Maintains gene-disease validity assessments and variant interpretation standards. Useful for understanding how variants are classified. -- **[ACMG SF v3.2 Gene List](https://www.nature.com/articles/s41436-023-02171-w)** — The 81 genes that ACMG recommends reporting incidental findings for. Note: CPSR uses its own cancer-focused panels (Genomics England PanelApp), not the ACMG SF list — CPSR covers 500+ cancer genes but omits non-cancer ACMG genes (cardiac, metabolic). A negative CPSR result does not constitute a complete ACMG incidental-findings screen. +- **[ACMG SF v3.3 Gene List](https://doi.org/10.1016/j.gim.2025.101454)** — The 84 genes that ACMG recommends reporting incidental findings for (Lee et al., Genet Med 2025;27(8):101454, doi:10.1016/j.gim.2025.101454). This is the list the pipeline's secondary-findings section applies (`assets/acmg_sf.tsv`). Note: CPSR uses its own cancer-focused panels (Genomics England PanelApp), not the ACMG SF list — CPSR covers 500+ cancer genes but omits non-cancer ACMG genes (cardiac, metabolic). A negative CPSR result does not constitute a complete ACMG incidental-findings screen. - **[MitoMap](https://www.mitomap.org/)** — The human mitochondrial genome database, with the curated disease status of mtDNA variants. Step 20 already puts ClinVar's record and gnomAD v3.1's frequency beside each call; MITOMAP is a manual lookup in a browser, as it answers scripted requests with a 403. - **[GeneReviews](https://www.ncbi.nlm.nih.gov/books/NBK1116/)** — Expert-authored, peer-reviewed descriptions of genetic conditions. If you find a pathogenic variant, the GeneReviews entry for the associated condition is the best single resource for understanding it. diff --git a/rust/crates/og-summary/src/collect/assets.rs b/rust/crates/og-summary/src/collect/assets.rs index a71eeb613..0c80fa8f1 100644 --- a/rust/crates/og-summary/src/collect/assets.rs +++ b/rust/crates/og-summary/src/collect/assets.rs @@ -294,4 +294,36 @@ mod tests { assert!(parse_alleles(Some("chr1:5:A"), false).is_err()); assert!(parse_alleles(None, true).unwrap().is_empty()); } + + /// The `#` lines of `acmg_sf.tsv` (its source, its columns, the note + /// on the quoted Table 1 text) are not table rows: the shipped file, + /// the same rows with no `#` line, and the same rows with more `#` + /// lines among them read as one table. + #[test] + fn acmg_header_lines_do_not_change_the_table() { + let repo = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../assets"); + let shipped = std::fs::read_to_string(format!("{repo}/acmg_sf.tsv")).unwrap(); + assert!(shipped.lines().any(|l| l.starts_with('#'))); + let rows: Vec<&str> = shipped + .split_inclusive('\n') + .filter(|l| !l.starts_with('#')) + .collect(); + let bare = rows.concat(); + let mut extra = String::from("# a header line\n"); + for (i, l) in rows.iter().enumerate() { + extra.push_str(l); + if i % 20 == 1 { + extra.push_str("# a line between rows\tAPC\tHGNC:583\n"); + } + } + let read = |tag: &str, body: &str| { + let d = crate::testutil::Dir::new(tag); + d.put("acmg_sf.tsv", body); + format!("{:?}", acmg_sf_rules(&d.0.display().to_string()).unwrap()) + }; + let want = format!("{:?}", acmg_sf_rules(repo).unwrap()); + assert_eq!(acmg_sf_rules(repo).unwrap().len(), 84); + assert_eq!(read("acmg-bare", &bare), want); + assert_eq!(read("acmg-extra", &extra), want); + } } From 7875c956f01af70cd103917e6599e67f8b2a151a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:04 +0100 Subject: [PATCH 41/86] docs(calling): name the source of the GRCh38 PAR intervals assets/par_grch38.bed stays without a header, because some readers take every line as an interval; the variant calling page names the Genome Reference Consortium's GRCh38 page and the coordinates. Bead: lic-19 --- docs/03-variant-calling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/03-variant-calling.md b/docs/03-variant-calling.md index 14c281141..2a4c23fe0 100644 --- a/docs/03-variant-calling.md +++ b/docs/03-variant-calling.md @@ -84,7 +84,7 @@ DeepVariant writes its outputs under `.part` names, and the script renames them ## Sex chromosomes -A male sample has one X and one Y. Outside the pseudoautosomal regions (PARs, the ends of X and Y that pair with each other), a heterozygous call on chrX or chrY cannot be real. With `male`, DeepVariant calls chrX and chrY haploid outside the PARs listed in `assets/par_grch38.bed` (the GRCh38 PAR1 and PAR2 of chrX and chrY), so those calls are homozygous. `female` and no sex call every contig diploid; no sex prints a note saying so. `run-all.sh` passes the sex it was given. +A male sample has one X and one Y. Outside the pseudoautosomal regions (PARs, the ends of X and Y that pair with each other), a heterozygous call on chrX or chrY cannot be real. With `male`, DeepVariant calls chrX and chrY haploid outside the PARs listed in `assets/par_grch38.bed` (the GRCh38 PAR1 and PAR2 of chrX and chrY), so those calls are homozygous. The PAR coordinates come from the [Genome Reference Consortium's GRCh38 page](https://www.ncbi.nlm.nih.gov/grc/human): PAR1 is chrX and chrY 10,001-2,781,479; PAR2 is chrX 155,701,383-156,030,895 and chrY 56,887,903-57,217,415 (1-based). The BED holds the same intervals with 0-based starts. It has no header line, because some of its readers take every line as an interval. `female` and no sex call every contig diploid; no sex prints a note saying so. `run-all.sh` passes the sex it was given. The alternative callers (03a GATK, 03b FreeBayes, 03d Octopus) call every contig diploid, chrX and chrY of a male sample too. Clair3 (03e, long reads) takes the same `male` or `female` argument (`--gender`). From d58805eb2b4ad403de09b42e9ec7b34c0f7137c2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:05 +0100 Subject: [PATCH 42/86] docs(changelog): note the licence and ACMG citation fixes Bead: lic-01 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..7dcc474de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The contribution terms and the guides now state one licence for OwnGenome: GPL-3.0-or-later. CONTRIBUTING.md granted contributions under GPL-3.0 only, and CLAUDE.md, the traits page and the Nextflow page said GPL-3.0; they now say GPL-3.0-or-later like README and `rust/Cargo.toml`. README carries the GPL's or-later notice, and the `nextflow.config` manifest names the licence. No output changes. + - The resources page named ACMG SF v3.2 with 81 genes; it now names v3.3 with 84 genes and the 2025 paper, the list `assets/acmg_sf.tsv` applies. + - `assets/acmg_sf.tsv` and the clinical filter page say that the `table_text` column holds short cited quotations of Table 1's reporting rule, and that "(See text)" points to the paper's section on the DCM genes. A new `og-summary` test shows that `#` lines do not change the table read from the file. + - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. From 207283a63039b37e027677018eb9d9ceee9c8542 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:06 +0100 Subject: [PATCH 43/86] fix(clinvar): keep ClinVar's condition on each hit Step 06 copied only ID, GENEINFO, CLNSIG and CLNREVSTAT onto each hit, so later steps knew the gene but not the disease ClinVar asserted. Both mirrors now also copy CLNDN and CLNDISDB when the ClinVar file's header defines them (every release does); a hand-built file without them runs as before, with a notice. The hits TSV and the output contract stay as they are. Part of #411 Bead: i411-hits-keep-condition --- CHANGELOG.md | 2 ++ docs/06-clinvar-screen.md | 6 +++--- modules/local/clinvar_screen/main.nf | 15 +++++++++++++-- scripts/06-clinvar-screen.sh | 17 +++++++++++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..2f5e6340c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Each ClinVar hit now keeps the condition ClinVar names for it. Step 06 copied only ClinVar's ID, gene, significance and review status onto each hit, so a later step knew the gene but not the disease: a RET hit could be Hirschsprung disease or MEN2. The hits VCF (`clinvar/_clinvar_hits.vcf`) now also carries `CLNDN`, ClinVar's disease name, and `CLNDISDB`, its disease database ids, in both the bash step and the Nextflow module. Two INFO fields are added; nothing is renamed or removed, and `clinvar/_clinvar_hits.tsv` is unchanged. A hand-built ClinVar file that does not define the two fields still runs, without them and with a notice. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index 9a608b5ac..3a8e5873a 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -53,7 +53,7 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. 4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. -6. Keeps the sample's records and combined alleles whose allele is in ClinVar (`bcftools isec -n=2 -w1`), copies ClinVar's `ID`, `GENEINFO`, `CLNSIG` and `CLNREVSTAT` onto them (`bcftools annotate --pair-logic exact`), and keeps only those whose genotype carries an ALT allele (`bcftools view -i 'GT~"[1-9]"'`, a non-zero allele index anywhere in the genotype). The match is by allele, so without this a `0/0` or `./.` record, the `0/0` half of a split multiallelic record and a reference-only row with ALT `.` would be listed as hits. A half call such as `./1` carries the ALT allele and is a hit; `GT="alt"` would drop it. The filter runs after the split, where it sees each allele's own genotype. No command line goes into the hits file's header (`--no-version`), so ClinVar's path on your machine stays out of it. +6. Keeps the sample's records and combined alleles whose allele is in ClinVar (`bcftools isec -n=2 -w1`), copies ClinVar's `ID`, `GENEINFO`, `CLNSIG`, `CLNREVSTAT`, `CLNDN` and `CLNDISDB` onto them (`bcftools annotate --pair-logic exact`), and keeps only those whose genotype carries an ALT allele (`bcftools view -i 'GT~"[1-9]"'`, a non-zero allele index anywhere in the genotype). The match is by allele, so without this a `0/0` or `./.` record, the `0/0` half of a split multiallelic record and a reference-only row with ALT `.` would be listed as hits. A half call such as `./1` carries the ALT allele and is a hit; `GT="alt"` would drop it. The filter runs after the split, where it sees each allele's own genotype. No command line goes into the hits file's header (`--no-version`), so ClinVar's path on your machine stays out of it. `CLNDN` is the condition ClinVar names for the allele and `CLNDISDB` its ids in disease databases such as MONDO, OMIM and MedGen. A gene can cause more than one disease, so a later step reads these to know which one ClinVar asserted. Every ClinVar release defines both. `bcftools annotate` stops on a tag the ClinVar file does not define, so a hand-built ClinVar file without them gives hits without them, and the step prints a notice. 7. Marks the hits that belong together. A matched combined allele is a hit of its own and carries `INFO/PGP_JOINED_FROM`, its two parts as `CHROM:POS:REF:ALT`. Each of its parts that is a hit too keeps its row and gets `INFO/PGP_PART_OF`, the combined record it belongs to, so the reader sees one allele with the combined record's own review status, not two unrelated hits. Two hits in one gene that could be one allele by the rule of item 5 but matched no combined record each get `INFO/PGP_ONE_ALLELE`, the other hit: possibly one allele, check ClinVar for a combined record. 8. Writes a call floor other than the default into the hits file's header, for example `##PGP_CALL_FLOOR=GQ=20,DP=8` (below). At the default the file has no such line and reads as GQ 30 and DP 10. 9. Writes the always-report list (below): the sample's call at every site of `assets/deep_intronic_sites.tsv`. @@ -100,8 +100,8 @@ The [runtime table](hardware-requirements.md#runtime-per-step) gives the measure | File | Description | |---|---| -| `clinvar/${SAMPLE}_clinvar_hits.vcf` | **The hits: your records that carry a ClinVar Pathogenic/Likely_pathogenic allele, with ClinVar's ID, gene (`GENEINFO`), significance (`CLNSIG`) and review status (`CLNREVSTAT`). Your genotype, GQ and DP are in the sample column, and a `##PGP_CALL_FLOOR` header line holds the call floor of a run that changed it.** A combined allele of two neighbouring records is a record of its own, with `PGP_JOINED_FROM`; its parts carry `PGP_PART_OF`, and two hits that may be one allele carry `PGP_ONE_ALLELE` | -| `clinvar/${SAMPLE}_clinvar_hits.tsv` | The same hits, one row each: `chrom`, `pos`, `ref`, `alt`, `genotype`, `clinvar_id`, `geneinfo`, `clnsig`, `clnrevstat` | +| `clinvar/${SAMPLE}_clinvar_hits.vcf` | **The hits: your records that carry a ClinVar Pathogenic/Likely_pathogenic allele, with ClinVar's ID, gene (`GENEINFO`), significance (`CLNSIG`), review status (`CLNREVSTAT`) and condition (`CLNDN`, with its disease database ids in `CLNDISDB`). Your genotype, GQ and DP are in the sample column, and a `##PGP_CALL_FLOOR` header line holds the call floor of a run that changed it.** A combined allele of two neighbouring records is a record of its own, with `PGP_JOINED_FROM`; its parts carry `PGP_PART_OF`, and two hits that may be one allele carry `PGP_ONE_ALLELE` | +| `clinvar/${SAMPLE}_clinvar_hits.tsv` | The same hits, one row each: `chrom`, `pos`, `ref`, `alt`, `genotype`, `clinvar_id`, `geneinfo`, `clnsig`, `clnrevstat`. The condition is in the hits VCF only | | `clinvar/${SAMPLE}_clinvar_cdna_check.tsv` | The cDNA read check, one row per hit: `chrom`, `pos`, `ref`, `alt`, `alt_reads`, `alt_clipped`, `ref_reads`, `ref_clipped` (the reads with each allele, and how many of them are clipped at an exon edge), `cdna_check` (`flagged`, `clear` or `not_checked`), `cdna_note` (the flag text or the reason it was not checked) and `construct_site` (the gene and name of a known lab-construct site, empty otherwise) | | `clinvar/${SAMPLE}_clinvar_cdna_scan.tsv` | The cDNA scan: a `# cdna_scan:` line, then one row per listed locus: `chrom`, `start` and `end` (the first junction's exon end and the last junction's exon start, 1-based), `junctions`, `reads` (distinct reads, so a read across two junctions counts once) and `junction_list` (`end-start:reads` per junction) | | `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv` | The always-report list, one row per site: the site's `chrom`, `pos`, `ref`, `alt`, `gene`, `hgvs` and `clinvar_id`, then `call` and `detail` | diff --git a/modules/local/clinvar_screen/main.nf b/modules/local/clinvar_screen/main.nf index 65f006875..f8241cedb 100644 --- a/modules/local/clinvar_screen/main.nf +++ b/modules/local/clinvar_screen/main.nf @@ -6,7 +6,8 @@ PASS variants against the ClinVar pathogenic subset. Writes _clinvar_hits.vcf: the sample's matching records the person carries (genotype with an ALT allele) with ClinVar's ID, GENEINFO, CLNSIG and - CLNREVSTAT copied on; _clinvar_hits.tsv: the same hits, one row each; + CLNREVSTAT copied on, and ClinVar's condition (CLNDN, CLNDISDB) when the + ClinVar file defines it; _clinvar_hits.tsv: the same hits, one row each; and prints the hits per ClinVar review-status star tier. Both files are read only on the contigs they share with each other and @@ -177,10 +178,20 @@ process CLINVAR_SCREEN { # Last, the tag pass marks each part of a matched combined allele # (PGP_PART_OF), and two hits in one gene that can be one allele but # matched no combined record (PGP_ONE_ALLELE). + # The condition goes on too, as scripts/06 copies it: CLNDN (ClinVar's + # disease name) and CLNDISDB (its disease database ids), so a later step + # can tell which of a gene's diseases ClinVar asserted. bcftools annotate + # stops on a tag the ClinVar file does not define, so each is copied only + # when the file's header defines it (every ClinVar release does). + clinvar_cols="ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT\$(bcftools view -h "\${CLINVAR_NORM}" | awk '/^##INFO= '$(cpath "${HITS}.part")' rm -f '$(cpath "${HITS}.untagged")' From 3c8f08110964e316fa24d7723b2811d43e92d814 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:07 +0100 Subject: [PATCH 44/86] test(clinvar): fake-docker case for the condition on step 06 hits Runs scripts/06 against a fake docker with a ClinVar header that defines CLNDN and CLNDISDB and one that does not, and runs the module's column line against the same two headers. Red on main in both halves. Part of #411 Bead: i411-hits-keep-condition --- tests/fake-docker/i411-hits-keep-condition.sh | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 tests/fake-docker/i411-hits-keep-condition.sh diff --git a/tests/fake-docker/i411-hits-keep-condition.sh b/tests/fake-docker/i411-hits-keep-condition.sh new file mode 100755 index 000000000..851bc9dec --- /dev/null +++ b/tests/fake-docker/i411-hits-keep-condition.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Step 06 copies ClinVar's condition onto each hit, in both mirrors: +# - scripts/06-clinvar-screen.sh: the bcftools annotate that writes the hits +# VCF copies INFO/CLNDN and INFO/CLNDISDB with ID, GENEINFO, CLNSIG and +# CLNREVSTAT when ClinVar's header defines them (every ClinVar release +# does). Without them a later step knows the gene but not which of its +# diseases ClinVar asserted for the allele; +# - a hand-built ClinVar file without them still runs: bcftools annotate +# stops on a tag the file does not define, so they are left out, with a +# notice; +# - modules/local/clinvar_screen/main.nf builds the same column list from +# the same header and hands it to its own bcftools annotate. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" + +export GENOME_DIR="${CASE_WORK}/genome" +seed_reference "$GENOME_DIR" +seed_clinvar "$GENOME_DIR" +seed_sample "$GENOME_DIR" sample1 +use_output_hook + +# ClinVar's own header lines for the two tags, from a ClinVar release. +CLNDN_LINE="##INFO=" +CLNDISDB_LINE='##INFO=' +BASE_HEADER='##fileformat=VCFv4.1 +##INFO= +##INFO= +##INFO=' +CHROM_LINE=$(printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO') + +# The answers step 06 reads from bcftools: one contig with records on both +# sides, a PASS record, record counts, and ClinVar's header from +# ${CASE_WORK}/clinvar-header.txt. Every call also gets its outputs created. +cat > "${CASE_WORK}/hook-step06" <<'HOOK' +#!/usr/bin/env bash +"${CASE_WORK:?}/hook-outputs" "$@" || exit $? +shift +case "$*" in + *"index -s "*) printf 'chr1\t248956422\t3\n' ;; + *"index -n "*) echo 3 ;; + *"view -H -f PASS "*) echo 1 ;; + *"view -h "*clinvar_pathogenic_chr.norm.vcf.gz*) cat "${CASE_WORK}/clinvar-header.txt" ;; + *"view -h "*) printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n' ;; +esac +exit 0 +HOOK +chmod +x "${CASE_WORK}/hook-step06" +export FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/hook-step06" + +# The step stops later, at an output the fake docker does not write (an -o +# relative to -w), so its exit code is not checked: the annotate call +# comes before it. +# annotate_cols: the -c list of the hits VCF's bcftools annotate in the docker log. +annotate_cols() { + awk '/^run / && /bcftools annotate / && /pair-logic exact/' "$FAKE_DOCKER_LOG" \ + | grep -oE -- "-c[^ ]* [^ ]*INFO/CLNREVSTAT[^ ]*" | head -n 1 +} + +# --- scripts/06: a ClinVar release defines both tags ------------------------------ +printf '%s\n%s\n%s\n%s\n' "$BASE_HEADER" "$CLNDN_LINE" "$CLNDISDB_LINE" "$CHROM_LINE" > "${CASE_WORK}/clinvar-header.txt" +: > "$FAKE_DOCKER_LOG" +run_rc step06-release "${SCRIPTS}/06-clinvar-screen.sh" sample1 +COLS=$(annotate_cols) +echo "annotate columns: ${COLS:-none}" +[ -n "$COLS" ] || fail "step 06 ran no bcftools annotate for the hits VCF" +grep -qF 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT,INFO/CLNDN,INFO/CLNDISDB' <<<"$COLS" \ + || fail "step 06 does not copy ClinVar's condition (INFO/CLNDN, INFO/CLNDISDB) onto the hits: ${COLS}" +output_lacks step06-release 'does not define CLNDN' + +# --- scripts/06: a hand-built ClinVar file without them -------------------------- +printf '%s\n%s\n' "$BASE_HEADER" "$CHROM_LINE" > "${CASE_WORK}/clinvar-header.txt" +rm -rf "${GENOME_DIR}/sample1/clinvar" +: > "$FAKE_DOCKER_LOG" +run_rc step06-handbuilt "${SCRIPTS}/06-clinvar-screen.sh" sample1 +COLS=$(annotate_cols) +echo "annotate columns: ${COLS:-none}" +[ -n "$COLS" ] || fail "step 06 ran no bcftools annotate for a ClinVar file without CLNDN" +if grep -q 'CLNDN\|CLNDISDB' <<<"$COLS"; then + fail "step 06 asks bcftools annotate for CLNDN or CLNDISDB, which this ClinVar file does not define (bcftools stops): ${COLS}" +fi +grep -qF 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT' <<<"$COLS" || fail "step 06 lost ClinVar's ID, gene, significance or review status: ${COLS}" +output_has step06-handbuilt 'does not define CLNDN and CLNDISDB' + +# --- the Nextflow module: the same columns, from the same header ------------------ +# Its clinvar_cols line, with the Groovy escapes undone, runs with a bcftools +# that prints the header; its annotate must take that list. +MOD="${REPO_ROOT}/modules/local/clinvar_screen/main.nf" +LINE=$(awk '/^process CLINVAR_SCREEN \{/ { on = 1 } on && /^ *clinvar_cols=/ { print; exit }' "$MOD") +[ -n "$LINE" ] || fail "the ClinVar screen module builds no clinvar_cols list for bcftools annotate" +LINE=$(sed -e 's/\\\$/$/g' -e 's/\\\\/\\/g' <<<"$LINE") +# shellcheck disable=SC2016 # the module's text, Groovy escape included +awk '/^process CLINVAR_SCREEN \{/ { on = 1 } on && /bcftools annotate .*pair-logic exact/ { getline; print; exit }' "$MOD" \ + | grep -qF -- '-c "\${clinvar_cols}"' || fail "the module's bcftools annotate does not take clinvar_cols" +# module_cols HEADER_FILE: the list the module's line builds from that header. +module_cols() { + # shellcheck disable=SC2034 # read by the module's line + CLINVAR_NORM=clinvar.norm.vcf.gz HEADER=$1 bash -c 'set -euo pipefail; bcftools() { cat "$HEADER"; }; eval "$1"; printf "%s" "$clinvar_cols"' _ "$LINE" +} +printf '%s\n%s\n%s\n%s\n' "$BASE_HEADER" "$CLNDISDB_LINE" "$CLNDN_LINE" "$CHROM_LINE" > "${CASE_WORK}/release-header.txt" +printf '%s\n%s\n' "$BASE_HEADER" "$CHROM_LINE" > "${CASE_WORK}/handbuilt-header.txt" +GOT=$(module_cols "${CASE_WORK}/release-header.txt") +[ "$GOT" = 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT,INFO/CLNDN,INFO/CLNDISDB' ] \ + || fail "the module copies '${GOT}' from a ClinVar release, want ClinVar's condition too (INFO/CLNDN, INFO/CLNDISDB)" +GOT=$(module_cols "${CASE_WORK}/handbuilt-header.txt") +[ "$GOT" = 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT' ] \ + || fail "the module copies '${GOT}' from a ClinVar file without CLNDN and CLNDISDB, want only ID, GENEINFO, CLNSIG and CLNREVSTAT" +echo "both mirrors copy ClinVar's condition when ClinVar defines it: PASS" From cd801724a91799a62122ae5dbc36155497654054 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:08:47 +0100 Subject: [PATCH 45/86] fix(roh): stream chromosome scope and emit neutral region hints Use assessed autosomes for the length denominator. Require per-site evidence on all 22 autosomes for genome-wide labels. Embed the one attributed imprinting asset and keep legacy parent keys as placeholders. Bead: lin-10 Bead: lin-13 Bead: lin-15 Bead: lin-16 Bead: lin-17 Bead: lic-41 --- assets/imprinted_regions.tsv | 21 +- .../og-summary/src/collect/group_c/roh.rs | 296 +++++++++--------- .../og-summary/src/collect/group_c/tests.rs | 63 +++- 3 files changed, 217 insertions(+), 163 deletions(-) diff --git a/assets/imprinted_regions.tsv b/assets/imprinted_regions.tsv index c71bb24dd..b55146765 100644 --- a/assets/imprinted_regions.tsv +++ b/assets/imprinted_regions.tsv @@ -1,14 +1,17 @@ -# Imprinting disorders caused by uniparental disomy (UPD), one row per chromosome or region. -# Read by og summary collect (IMPRINTED_REGIONS in rust/crates/og-summary holds the same rows; its -# tests check that the two agree) to name what a run of homozygosity over most of one chromosome could mean. -# start and end: GRCh38, 0-based start, exclusive end, the UCSC cytoBand.hg38.txt bands of the region; -# empty for a row that applies to the whole chromosome. A row with a region applies when a run on the -# flagged chromosome overlaps it. "-" is no known imprinting disorder for UPD from that parent. -# Overview: Eggermann et al., Imprinting disorders, Nat Rev Dis Primers 2023 (PMID 37386011). +# Imprinted chromosomes and regions. Factual reference rows, curated by OwnGenome. +# Sole source: embedded by og summary collect at build time. The UPD hint emits region and PMID only; +# maternal_upd and paternal_upd remain reference facts here, never a diagnosis from a ROH match. +# start and end are retained for format compatibility and are empty: the hint is chromosome-wide. +# Mixed iso/heterodisomy may leave the imprinted locus outside the observed homozygous runs. +# Region labels are broad cytobands, not diagnostic locus coordinates or a specialist-reviewed assay. +# Overview: Eggermann et al., Imprinting disorders, Nat Rev Dis Primers 9, 33 (2023). +# https://doi.org/10.1038/s41572-023-00443-4 (PMID 37386011). +# Mechanism: Robinson, BioEssays 22:452-459 (2000), https://pubmed.ncbi.nlm.nih.gov/10797485/. +# Per-row PMIDs support the parent-of-origin reference facts; names and symbols are factual labels. chrom start end region maternal_upd paternal_upd pmid chr6 6q24 (PLAGL1, HYMAI) - Transient neonatal diabetes mellitus (6q24-related) 20301706 chr7 7p12 (GRB10), 7q32 (MEST) Silver-Russell syndrome - 27585961 -chr11 0 22000000 11p15 (H19/IGF2, KCNQ1OT1) Silver-Russell syndrome Beckwith-Wiedemann syndrome 29377879;27585961 +chr11 11p15 (H19/IGF2, KCNQ1OT1) Silver-Russell syndrome Beckwith-Wiedemann syndrome 29377879;27585961 chr14 14q32.2 (DLK1-MEG3) Temple syndrome Kagami-Ogata syndrome 24891339;26377239 -chr15 20500000 33400000 15q11-q13 (SNRPN, UBE3A) Prader-Willi syndrome Angelman syndrome 22237428;27615419 +chr15 15q11-q13 (SNRPN, UBE3A) Prader-Willi syndrome Angelman syndrome 22237428;27615419 chr20 20q13.32 (GNAS) Mulchandani-Bhoj-Conlin syndrome (growth failure) Pseudohypoparathyroidism type 1B 26248010;29959430 diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index f0fb24dc4..2724c416d 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -51,6 +51,54 @@ pub const MASKED_BANDS: [(&str, i128, i128); 25] = [ /// GRCh38's chr1 to chr22 (UCSC hg38.chrom.sizes). pub const AUTOSOME_BP: i128 = 2_875_001_522; + +// UCSC hg38.chrom.sizes, in chromosome order. ST rows tell us which +// chromosome lengths belong in this run's denominator, not callable bases. +const AUTOSOME_LENGTHS: [i128; 22] = [ + 248_956_422, + 242_193_529, + 198_295_559, + 190_214_555, + 181_538_259, + 170_805_979, + 159_345_973, + 145_138_636, + 138_394_717, + 133_797_422, + 135_086_622, + 133_275_309, + 114_364_328, + 107_043_718, + 101_991_189, + 90_338_345, + 83_257_441, + 80_373_285, + 58_617_616, + 64_444_167, + 46_709_983, + 50_818_468, +]; + +fn autosome_index(chrom: &str) -> Option { + is_autosome(chrom).then(|| chrom[3..].parse::().unwrap() - 1) +} + +fn covered_unmasked_bp(covered: &[bool; 22]) -> i128 { + covered + .iter() + .enumerate() + .filter(|(_, yes)| **yes) + .map(|(i, _)| { + let chrom = format!("chr{}", i + 1); + AUTOSOME_LENGTHS[i] + - MASKED_BANDS + .iter() + .filter(|(c, ..)| *c == chrom) + .map(|(_, a, b)| b - a) + .sum::() + }) + .sum() +} /// The runs F_ROH and the one-chromosome flag count (docs/11-roh-analysis.md). pub const ROH_MIN_BP: f64 = 5e6; /// One chromosome carrying most of the long runs: at least this much of it @@ -123,87 +171,31 @@ pub fn roh_band(f: f64) -> &'static str { .map_or("parent and child, full siblings (F 1/4)", |b| b.1) } -/// assets/imprinted_regions.tsv, the same rows (a test keeps the two -/// equal): chrom, the region's span (none for the whole chromosome), the -/// region, the maternal and paternal UPD syndromes and the PMIDs. -/// chrom, span, then four text fields. -type Imprinted = ( - &'static str, - Option<(i128, i128)>, - &'static str, - &'static str, - &'static str, - &'static str, -); +/// One source for region names and their citations. Embed the shipped +/// asset so a staged report needs no second runtime file or copied table. +const IMPRINTED_TSV: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../assets/imprinted_regions.tsv" +)); -pub const IMPRINTED_REGIONS: [Imprinted; 6] = [ - ( - "chr6", - None, - "6q24 (PLAGL1, HYMAI)", - "-", - "Transient neonatal diabetes mellitus (6q24-related)", - "20301706", - ), - ( - "chr7", - None, - "7p12 (GRB10), 7q32 (MEST)", - "Silver-Russell syndrome", - "-", - "27585961", - ), - ( - "chr11", - Some((0, 22_000_000)), - "11p15 (H19/IGF2, KCNQ1OT1)", - "Silver-Russell syndrome", - "Beckwith-Wiedemann syndrome", - "29377879;27585961", - ), - ( - "chr14", - None, - "14q32.2 (DLK1-MEG3)", - "Temple syndrome", - "Kagami-Ogata syndrome", - "24891339;26377239", - ), - ( - "chr15", - Some((20_500_000, 33_400_000)), - "15q11-q13 (SNRPN, UBE3A)", - "Prader-Willi syndrome", - "Angelman syndrome", - "22237428;27615419", - ), - ( - "chr20", - None, - "20q13.32 (GNAS)", - "Mulchandani-Bhoj-Conlin syndrome (growth failure)", - "Pseudohypoparathyroidism type 1B", - "26248010;29959430", - ), -]; - -/// The imprinted regions a flagged chromosome joins: every row of the -/// whole chromosome, and a row with a region only when a run overlaps it. -fn imprinted(chrom: &str, runs: &[(i128, i128)]) -> Value { +/// A chromosome hint cannot determine parental origin or diagnose a +/// syndrome. The two legacy keys stay as placeholders for old readers. +fn imprinted(chrom: &str) -> Value { Value::Array( - IMPRINTED_REGIONS - .iter() - .filter(|r| r.0 == chrom) - .filter(|r| { - r.1.is_none_or(|(a, b)| runs.iter().any(|&(s, e)| overlap(a, b, s, e) > 0)) - }) - .map(|r| { - Obj::new() - .set("region", s(r.2)) - .set("maternal_upd", s(r.3)) - .set("paternal_upd", s(r.4)) - .set("pmid", s(r.5)) - .val() + IMPRINTED_TSV + .lines() + .filter(|line| !line.starts_with('#')) + .skip(1) + .filter_map(|line| { + let f: Vec<&str> = line.split('\t').collect(); + (f.len() == 7 && f[0] == chrom).then(|| { + Obj::new() + .set("region", s(f[3])) + .set("maternal_upd", s("-")) + .set("paternal_upd", s("-")) + .set("pmid", s(f[6])) + .val() + }) }) .collect(), ) @@ -228,11 +220,25 @@ pub fn section(c: &Ctx<'_>) -> Out { let (mut total, mut largest, mut masked) = (0.0_f64, 0.0_f64, 0.0_f64); let (mut segs, mut sex_segs) = (0usize, 0usize); let mut big = Vec::new(); - // chrom -> the bases of its long runs, and the runs (start, end) - let mut per_chrom: indexmap::IndexMap)> = - indexmap::IndexMap::new(); - for line in lines(&p.path)? { - let c = pycompat::text::split(&line, None, None); + // chrom -> the bases of its long runs + let mut per_chrom: indexmap::IndexMap = indexmap::IndexMap::new(); + let (mut assessed, mut observed) = ([false; 22], [false; 22]); + let mut reader = crate::text_lines(&p.path)?; + let mut line = String::new(); + loop { + line.clear(); + if !reader.read_line(&mut line).map_err(|e| e.to_string())? { + break; + } + let c: Vec<&str> = line.split_whitespace().collect(); + if c.len() >= 6 + && c[0] == "ST" + && let Some(i) = autosome_index(c[2]) + && crate::int(c[3]).is_ok_and(|pos| pos > 0 && pos <= AUTOSOME_LENGTHS[i]) + && matches!(c[4], "0" | "1") + { + assessed[i] = true; + } if c.len() < 6 || c[0] != "RG" { continue; } @@ -257,6 +263,7 @@ pub fn section(c: &Ctx<'_>) -> Out { if !is_autosome(c[2]) { continue; } + observed[autosome_index(c[2]).unwrap()] = true; let inside = length.min(masked_overlap(c[2], start, end) as f64); masked += inside; length -= inside; @@ -268,8 +275,7 @@ pub fn section(c: &Ctx<'_>) -> Out { largest = largest.max(length); if length >= ROH_MIN_BP { let pc = per_chrom.entry(c[2].to_owned()).or_default(); - pc.0 += length; - pc.1.push((start, end)); + *pc += length; let mut row = Obj::new() .set("region", s(&format!("{}:{}-{}", c[2], c[3], c[4]))) .set("mb", r1(length / 1e6, 1)); @@ -279,19 +285,30 @@ pub fn section(c: &Ctx<'_>) -> Out { big.push(row.val()); } } - let long_bp: f64 = per_chrom.values().fold(0.0, |a, v| a + v.0); - let f_roh = long_bp / autosome_unmasked_bp() as f64; + let long_bp: f64 = per_chrom.values().sum(); + let has_sites = assessed.iter().any(|yes| *yes); + // Old -O r files omit ST rows. Their runs give a lower bound on + // chromosome scope, never evidence for a genome-wide interpretation. + let covered = std::array::from_fn(|i| assessed[i] || observed[i]); + let denominator = covered_unmasked_bp(&covered); + let complete = has_sites + && assessed.iter().all(|yes| *yes) + && denominator >= autosome_unmasked_bp() + && observed + .iter() + .zip(assessed) + .all(|(run, site)| !run || site); + let f_roh = (denominator > 0).then(|| long_bp / denominator as f64); let mut upd = Value::Null; - // The first chromosome of the most bases, as max() picks. - let top = - per_chrom.iter().fold( - None::<(&String, &(f64, Vec<(i128, i128)>))>, - |best, kv| match best { - Some(b) if b.1.0 >= kv.1.0 => Some(b), - _ => Some(kv), - }, - ); - if let Some((chrom, (bp, runs))) = top + // Retain the first chromosome on ties, as before. + let top = per_chrom + .iter() + .fold(None::<(&String, &f64)>, |best, kv| match best { + Some(b) if b.1 >= kv.1 => Some(b), + _ => Some(kv), + }); + if complete + && let Some((chrom, bp)) = top && *bp >= UPD_MIN_BP && *bp > UPD_SHARE * long_bp { @@ -299,11 +316,12 @@ pub fn section(c: &Ctx<'_>) -> Out { .set("chrom", s(chrom)) .set("mb", r1(bp / 1e6, 1)) .set("share", r1(bp / long_bp, 2)) - .set("imprinted", imprinted(chrom, runs)) + .set("imprinted", imprinted(chrom)) + .set("note", s("Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH.")) .val(); } - let relationship = if upd == Value::Null { - s(roh_band(f_roh)) + let relationship = if complete && upd == Value::Null { + s(roh_band(f_roh.unwrap_or(0.0))) } else { Value::Null }; @@ -315,9 +333,20 @@ pub fn section(c: &Ctx<'_>) -> Out { .set("masked_mb", r1(masked / 1e6, 1)) .set("sex_chromosome_segments", count(sex_segs)) .set("long_mb", r1(long_bp / 1e6, 1)) - .set("f_roh", r1(f_roh, 4)) + .set("f_roh", f_roh.map_or(Value::Null, |f| r1(f, 4))) .set("relationship", relationship) - .set("upd_flag", upd); + .set("upd_flag", upd) + .set("scope", Obj::new() + .set("source", s(if has_sites { "sites" } else { "segments_only" })) + .set("autosomes", Value::Array(covered.iter().enumerate() + .filter(|(_, yes)| **yes).map(|(i, _)| s(&format!("chr{}", i + 1))).collect())) + .set("unmasked_bp", Value::Int(denominator)) + .set("partial", Value::Bool(!complete)) + .set("note", s(if complete { + "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } else { + "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + })).val()); Ok(Some((p, data.val()))) } @@ -443,47 +472,30 @@ mod tests { use super::*; use crate::testutil::Dir; - /// assets/imprinted_regions.tsv holds the rows of [`IMPRINTED_REGIONS`] - /// (tests/test_collect_summary.py checked the Python copy the same way). #[test] - fn imprinted_regions_are_the_asset_rows() { - let path = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../assets/imprinted_regions.tsv" - ); - let text = std::fs::read_to_string(path).unwrap(); - let mut lines = text.lines().filter(|l| !l.starts_with('#')); - let head: Vec<&str> = lines.next().unwrap().split('\t').collect(); - let col = |name: &str| head.iter().position(|h| *h == name).unwrap(); - let num = |x: &str| (!x.is_empty()).then(|| x.parse::().unwrap()); - let rows: Vec<_> = lines - .map(|l| { - let f: Vec<&str> = l.split('\t').collect(); - let span = num(f[col("start")]).zip(num(f[col("end")])); - ( - f[col("chrom")].to_owned(), - span, - f[col("region")].to_owned(), - f[col("maternal_upd")].to_owned(), - f[col("paternal_upd")].to_owned(), - f[col("pmid")].to_owned(), - ) - }) - .collect(); - let want: Vec<_> = IMPRINTED_REGIONS - .iter() - .map(|r| { - ( - r.0.to_owned(), - r.1, - r.2.to_owned(), - r.3.to_owned(), - r.4.to_owned(), - r.5.to_owned(), - ) - }) + fn imprinted_asset_is_the_only_table() { + let rows: Vec<_> = IMPRINTED_TSV + .lines() + .filter(|l| !l.starts_with('#')) + .skip(1) .collect(); - assert_eq!(rows, want); + assert_eq!(rows.len(), 6); + for chrom in ["chr6", "chr7", "chr11", "chr14", "chr15", "chr20"] { + let Value::Array(hints) = imprinted(chrom) else { + panic!() + }; + assert_eq!(hints.len(), 1); + let row = rows + .iter() + .find(|line| line.starts_with(&format!("{chrom}\t"))) + .unwrap(); + let f: Vec<_> = row.split('\t').collect(); + assert_eq!(f.len(), 7); + assert!(f[6].split(';').all(|pmid| pmid.parse::().is_ok())); + assert!(f[1].is_empty() && f[2].is_empty()); + } + assert_eq!(AUTOSOME_LENGTHS.iter().sum::(), AUTOSOME_BP); + assert_eq!(covered_unmasked_bp(&[true; 22]), autosome_unmasked_bp()); } #[test] diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index b072af7cb..4f34ef10f 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -244,13 +244,19 @@ fn roh_scope_chr20_uses_its_own_unmasked_denominator() { assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["upd_flag"]), &Value::Null); assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(59_744_167)); - assert_eq!(at(&r, &["scope", "autosomes"]), &Value::Array(vec![Value::Str("chr20".into())])); + assert_eq!( + at(&r, &["scope", "autosomes"]), + &Value::Array(vec![Value::Str("chr20".into())]) + ); } #[test] fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { - let r = roh_scope_summary(&[22, 20, 20], "RG\tS\tchr20\t1000001\t9000000\t8000000\nRG\tS\tchr22\t20000001\t28000000\t8000000\n"); - assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1718)); + let r = roh_scope_summary( + &[22, 20, 20], + "RG\tS\tchr20\t1000001\t9000000\t8000000\nRG\tS\tchr22\t20000001\t28000000\t8000000\n", + ); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1717)); assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(93_162_635)); let no_runs = roh_scope_summary(&[20], ""); @@ -263,8 +269,14 @@ fn roh_scope_sparse_all_autosomes_keep_the_whole_genome_band() { let sites: Vec = (1..=22).collect(); let r = roh_scope_summary(&sites, "RG\tS\tchr2\t10000001\t18000000\t8000000\n"); assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.003)); - assert_eq!(at(&r, &["relationship"]), &Value::Str("no sign of related parents".into())); - assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(2_658_001_522)); + assert_eq!( + at(&r, &["relationship"]), + &Value::Str("no sign of related parents".into()) + ); + assert_eq!( + at(&r, &["scope", "unmasked_bp"]), + &Value::Int(2_658_001_522) + ); assert_eq!(at(&r, &["scope", "partial"]), &Value::Bool(false)); } @@ -274,7 +286,10 @@ fn roh_scope_a_segment_only_file_never_establishes_scope() { assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.3515)); assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["upd_flag"]), &Value::Null); - assert_eq!(at(&r, &["scope", "source"]), &Value::Str("segments_only".into())); + assert_eq!( + at(&r, &["scope", "source"]), + &Value::Str("segments_only".into()) + ); let empty = roh_scope_summary(&[], "# empty\n"); assert_eq!(at(&empty, &["f_roh"]), &Value::Null); } @@ -285,15 +300,39 @@ fn roh_scope_upd_is_a_region_hint_without_syndrome_names() { // Distal chr15 and chr11 runs do not establish the imprinted locus's // parental origin, but mixed iso/heterodisomy can involve that locus. for (chrom, start, end, length, region) in [ - (15, 40000001, 101991189, 61991189, "15q11-q13 (SNRPN, UBE3A)"), - (11, 60000001, 100000000, 40000000, "11p15 (H19/IGF2, KCNQ1OT1)"), + ( + 15, + 40000001, + 101991189, + 61991189, + "15q11-q13 (SNRPN, UBE3A)", + ), + ( + 11, + 60000001, + 100000000, + 40000000, + "11p15 (H19/IGF2, KCNQ1OT1)", + ), (20, 1000001, 22000000, 21000000, "20q13.32 (GNAS)"), ] { - let r = roh_scope_summary(&sites, &format!("RG\tS\tchr{chrom}\t{start}\t{end}\t{length}\n")); + let r = roh_scope_summary( + &sites, + &format!("RG\tS\tchr{chrom}\t{start}\t{end}\t{length}\n"), + ); let u = at(&r, &["upd_flag"]); - assert_eq!(at(u, &["imprinted", "0", "region"]), &Value::Str(region.into())); - assert_eq!(at(u, &["imprinted", "0", "maternal_upd"]), &Value::Str("-".into())); - assert_eq!(at(u, &["imprinted", "0", "paternal_upd"]), &Value::Str("-".into())); + assert_eq!( + at(u, &["imprinted", "0", "region"]), + &Value::Str(region.into()) + ); + assert_eq!( + at(u, &["imprinted", "0", "maternal_upd"]), + &Value::Str("-".into()) + ); + assert_eq!( + at(u, &["imprinted", "0", "paternal_upd"]), + &Value::Str("-".into()) + ); let note = text(at(u, &["note"])); assert!(note.contains("deletion") && note.contains("acquired") && note.contains("parent")); assert_eq!(at(&r, &["relationship"]), &Value::Null); From 0de4fd2b5696e0cd548b1892fcfeaabfea0b9618 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:12:53 +0100 Subject: [PATCH 46/86] fix(archive): verify complete read content before removing BAM Preserve explicit MD/NM tags and compare order-independent checksums of all records and tags in archive and restore. Keep the source on failed checks. Bead: plat-62 --- CHANGELOG.md | 2 + assets/output_contract.tsv | 2 + docs/34-cram-archive.md | 31 +++++--- modules/local/cram_archive/main.nf | 38 +++++++-- scripts/34-cram-archive.sh | 44 +++++++++-- scripts/ci/e2e-case-paths.tsv | 1 + scripts/ci/e2e-case-times.tsv | 1 + tests/e2e/qc-3-cram-checksum.sh | 115 ++++++++++++++++++++++++++++ tests/fake-docker/qc-steps-33-34.sh | 31 +++++++- 9 files changed, 241 insertions(+), 24 deletions(-) create mode 100755 tests/e2e/qc-3-cram-checksum.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..4e3a75bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/assets/output_contract.tsv b/assets/output_contract.tsv index 1d567bf17..16f876eac 100644 --- a/assets/output_contract.tsv +++ b/assets/output_contract.tsv @@ -57,6 +57,8 @@ file {sample}/aligned/{sample}_sorted.bam.bai 02 ALIGN_MINIMAP2 both index stabl file {sample}/aligned/{sample}_sorted.cram 34 CRAM_ARCHIVE both cram experimental - - file {sample}/aligned/{sample}_sorted.cram.crai 34 CRAM_ARCHIVE both index experimental - - file {sample}/aligned/{sample}_sorted.cram.flagstat 34 CRAM_ARCHIVE both bin experimental - - +file {sample}/aligned/{sample}_sorted.cram.checksum 34 CRAM_ARCHIVE both text experimental - - +file {sample}/aligned/{sample}_sorted.bam.checksum 34 - bash text experimental - - file {sample}/ancestry/{sample}_ancestry.tsv 25 PRS_SUMMARY both tsv stable - - file {sample}/ancestry_fine/{sample}_ancestry_fine.tsv 26b ANCESTRY_FINE_REPORT both tsv experimental - - file {sample}/ancestry_fine/{sample}_ancestry_fine.txt 26b ANCESTRY_FINE_REPORT nextflow text experimental - - diff --git a/docs/34-cram-archive.md b/docs/34-cram-archive.md index cefcf3d42..d4d74b062 100644 --- a/docs/34-cram-archive.md +++ b/docs/34-cram-archive.md @@ -1,20 +1,20 @@ # Step 34: CRAM Archive ## What This Does -Writes the sample's alignments as CRAM beside the BAM, checks that the CRAM holds exactly the same reads, and, only when you ask and only after that check passed, deletes the BAM. `--restore` writes the BAM back from the CRAM. +Writes the sample's alignments as CRAM beside the BAM and checks their read content. It deletes the BAM only when you ask, after the checks pass. `--restore` writes the BAM back from the CRAM. ## Why -A 30x genome keeps 80 to 120 GB of BAM after the analysis is done. A CRAM stores each read as its difference from the reference and is about half that size or less (on the CI fixture, 68 MB of CRAM for 133 MB of BAM), with nothing lost: every read, base quality, flag and tag is still there. Most people keep their alignments to rerun a step when a tool or database improves; a CRAM keeps that option for half the disk. +A 30x genome keeps 80 to 120 GB of BAM after the analysis is done. A CRAM stores each read as its difference from the reference and is about half that size or less. The CI fixture has 68 MB of CRAM for 133 MB of BAM. Read content is checked before the original can be deleted. Most people keep their alignments to rerun a step when a tool or database improves. A CRAM keeps that option for half the disk. The price is the reference: a CRAM can only be read with a FASTA that holds the same sequences as the one it was written with. Keep `reference/GRCh38_no_alt_analysis_set.fasta` (or your `REF_FASTA`) as long as you keep the CRAM. samtools checks each contig's MD5 when it reads a CRAM and stops with an error on a different file, so a wrong reference cannot give silently wrong reads. ## Tool -- **samtools** (`view -C`, `index`, `quickcheck`, `flagstat`) +- **samtools** (`view -C`, `index`, `quickcheck`, `flagstat`, `checksum`) ## Docker Image - `SAMTOOLS_IMAGE` -Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. +Pinned in `versions.env`. [Image versions](versions.md) lists the current tag. ## Command ```bash @@ -25,20 +25,26 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. The script runs, in the samtools container: ```bash -samtools view -C --reference "${REF_FASTA}" -o aligned/${SAMPLE}_sorted.part.cram aligned/${SAMPLE}_sorted.bam +samtools view -C --reference "${REF_FASTA}" --output-fmt-option store_md=1 --output-fmt-option store_nm=1 -o aligned/${SAMPLE}_sorted.part.cram aligned/${SAMPLE}_sorted.bam samtools quickcheck -v aligned/${SAMPLE}_sorted.part.cram samtools index aligned/${SAMPLE}_sorted.part.cram samtools flagstat aligned/${SAMPLE}_sorted.bam samtools flagstat --input-fmt-option reference="${REF_FASTA}" aligned/${SAMPLE}_sorted.part.cram +samtools checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' aligned/${SAMPLE}_sorted.bam +samtools checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' --input-fmt-option reference="${REF_FASTA}" --input-fmt-option decode_md=0 aligned/${SAMPLE}_sorted.part.cram ``` -The check is that the CRAM passes `samtools quickcheck` (a CRAM cut short has no end-of-file block) and that the two `flagstat` outputs are equal line for line: the same number of reads, mapped, paired, properly paired, duplicates and secondary and supplementary alignments. Only then does the CRAM lose its `.part` name. When the check fails, the CRAM is removed, the BAM is kept and the step exits non-zero, also with `--delete-bam`. An older CRAM is always written again rather than trusted, because the BAM may have been aligned again since. +The CRAM must pass `samtools quickcheck`, the two `flagstat` reports must match, and the content checksums must match. The checksum reads every record, including secondary and supplementary alignments. It covers read names, sequences, qualities, flags, positions, CIGARs, mate fields and all auxiliary tags. A changed base quality or tag can leave `flagstat` counts unchanged and still fail this check. A decoding error also fails the step. -`--restore` does the same in reverse: the BAM is written under a `.part` name, checked against the CRAM, and moved into place. It refuses to write over a BAM that exists. +The checksums use samtools' order-independent aggregation. The comparison ignores the filename header and sorts the report rows, so read and tag order do not affect it. samtools sanitizes format-level differences, such as an unmapped read's MAPQ and equivalent CIGARs. Explicit `MD` and `NM` tags are stored in the CRAM. They are not regenerated during the check or restore, so missing tags stay missing. These CRC-based content checks detect conversion faults. They are not a cryptographic identity proof. See the [samtools 1.24 checksum manual](https://www.htslib.org/doc/1.24/samtools-checksum.html). -One archive or restore of a sample runs at a time. Archive, `--delete-bam` and `--restore` all take the same lock, `aligned/${SAMPLE}_sorted.lock`, with `flock` before they read or write any of the sample's files. A second run on the same sample, in another terminal or as a Nextflow `CRAM_ARCHIVE` task on that BAM, stops at once with `ERROR: another archive or restore of is running` and changes nothing; run it again when the first has finished. Without the lock, two runs would delete or overwrite the `.part` files the other is writing and checking, and `--delete-bam` could remove a BAM the other run is still reading. The lock goes when the run that holds it ends, however it ends (an error, Ctrl-C or `kill -9`), so it is never left behind; the empty `.lock` file stays and means nothing on its own. `flock` comes with util-linux on Linux; on macOS install it with `brew install flock`, or the step warns and runs without the lock. +Only after all checks pass does the CRAM lose its `.part` name. A mismatch or a failed or empty checksum command removes the partial CRAM, keeps the BAM and its index, and exits non-zero. This also applies with `--delete-bam`. An older published CRAM stays in place if a new attempt fails. Every archive attempt writes a fresh CRAM from the current BAM before it can delete anything. The checksum adds a full read of each file to the archive and restore checks. -In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CRAM_ARCHIVE`) for every BAM of the run, except the rows given as CRAM, which have one already, and publishes `//aligned/_sorted.cram`. It never deletes anything: delete the BAM yourself once the CRAM is there, or run this script on it. It takes the same lock, next to the BAM the samplesheet row names (`.lock`), and fails the task while this script holds it. That exclusion between this script and a `CRAM_ARCHIVE` task holds on Linux, where the container shares the host kernel; on macOS with Docker Desktop only two runs of this script exclude each other. When it cannot write the lock file there, it warns and reads the BAM without the lock. +`--restore` does the same in reverse: the BAM is written under a `.part` name, checked against the CRAM, and moved into place. It refuses to write over a BAM that exists. A failed check removes the partial BAM and keeps the CRAM. + +One archive or restore of a sample runs at a time. Archive, `--delete-bam` and `--restore` all take the same lock, `aligned/${SAMPLE}_sorted.lock`, with `flock` before they read or write any of the sample's files. A second run on the same sample, from another terminal or a Nextflow `CRAM_ARCHIVE` task, stops at once with `ERROR: another archive or restore of is running`. It changes nothing. Run it again when the first has finished. Without the lock, two runs would delete or overwrite the `.part` files the other is writing and checking, and `--delete-bam` could remove a BAM the other run is still reading. The lock goes when its run ends, including on an error, Ctrl-C or `kill -9`. The empty `.lock` file stays and means nothing on its own. `flock` comes with util-linux on Linux. On macOS install it with `brew install flock`, or the step warns and runs without the lock. + +In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CRAM_ARCHIVE`) for every BAM of the run, except the rows given as CRAM, which have one already, and publishes `//aligned/_sorted.cram`. It never deletes anything: delete the BAM yourself once the CRAM is there, or run this script on it. It takes the same lock, next to the BAM the samplesheet row names (`.lock`), and fails the task while this script holds it. This script and a `CRAM_ARCHIVE` task exclude each other on Linux, where the container shares the host kernel. On macOS with Docker Desktop, only two runs of this script exclude each other. When it cannot write the lock file there, it warns and reads the BAM without the lock. ## Output Files | File | Description | @@ -46,6 +52,7 @@ In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CR | `aligned/${SAMPLE}_sorted.cram` | The alignments | | `aligned/${SAMPLE}_sorted.cram.crai` | Its index | | `aligned/${SAMPLE}_sorted.cram.flagstat`, `aligned/${SAMPLE}_sorted.bam.flagstat` | The two `flagstat` outputs the check compared | +| `aligned/${SAMPLE}_sorted.cram.checksum`, `aligned/${SAMPLE}_sorted.bam.checksum` | Normalized content checksum reports, without the filename header. Nextflow publishes only the CRAM report. | | `aligned/${SAMPLE}_sorted.lock` | The empty file the lock is taken on | ## Rerunning steps from a CRAM @@ -57,9 +64,9 @@ In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CR | VCF-only steps | Every one: steps 06, 07, 11, 12, 13, 14, 17, 22, 23, 25, 26, 27, 30 and 31 read the VCF, which this step never touches | ## Runtime -Writing a CRAM reads the whole BAM and compresses it again; the check reads both files once more. On CI's fixture slice it takes seconds; a whole genome has not been timed here. +Writing a CRAM reads the whole BAM and compresses it again. The checks read both files for flagstat and again for the checksum. On a synthetic 100,000-read BAM in the pinned image, three trials took 4.44 to 4.99 seconds with 44.1 to 45.3 MiB peak RSS. The count-only checks took 3.39 to 3.99 seconds with 36.3 to 36.5 MiB. These measurements use one thread under amd64 emulation and exclude container startup. A whole genome has not been timed here. ## Notes -- Keep the reference FASTA (and its `.fai`) with the CRAM. A FASTA decodes the CRAM only when every contig the CRAM uses has the same sequence; keeping the file you wrote it with is the sure way. +- Keep the reference FASTA (and its `.fai`) with the CRAM. A FASTA decodes the CRAM only when every contig the CRAM uses has the same sequence. Keep the file you wrote it with. - The CRAM is written with samtools' default CRAM version and compression, which every current htslib, Picard and GATK read. -- A CRAM made elsewhere (nf-core/sarek writes them) can go straight into the Nextflow samplesheet; for the bash steps, `--restore` needs it at `aligned/${SAMPLE}_sorted.cram` with its `.crai`. +- A CRAM made elsewhere (nf-core/sarek writes them) can go straight into the Nextflow samplesheet. For the bash steps, `--restore` needs it at `aligned/${SAMPLE}_sorted.cram` with its `.crai`. diff --git a/modules/local/cram_archive/main.nf b/modules/local/cram_archive/main.nf index a845b997c..c9b96c270 100644 --- a/modules/local/cram_archive/main.nf +++ b/modules/local/cram_archive/main.nf @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CRAM_ARCHIVE writes _sorted.cram (+ .crai) beside the published BAM and fails the task unless the CRAM passes `samtools quickcheck` and its - `samtools flagstat` equals the BAM's line for line. It never deletes the + `samtools flagstat` and full-content checksums equal the BAM's. It never deletes the BAM: delete it yourself once the CRAM is there, or use scripts/34-cram-archive.sh --delete-bam. A CRAM can only be read with the reference it was written with: keep that FASTA. It takes the same lock as @@ -32,6 +32,7 @@ process CRAM_ARCHIVE { output: tuple val(meta), path("${meta.id}_sorted.cram"), path("${meta.id}_sorted.cram.crai"), emit: cram tuple val(meta), path("${meta.id}_sorted.cram.flagstat"), emit: flagstat + tuple val(meta), path("${meta.id}_sorted.cram.checksum"), emit: checksum path "versions.yml", emit: versions when: @@ -56,7 +57,7 @@ process CRAM_ARCHIVE { else echo "WARNING: could not take the lock \$lock; nothing stops scripts/34-cram-archive.sh from changing ${bam} during this task." >&2 fi - samtools view -@ ${task.cpus} -C --reference ${reference} -o ${meta.id}_sorted.cram ${bam} + samtools view -@ ${task.cpus} -C --reference ${reference} --output-fmt-option store_md=1 --output-fmt-option store_nm=1 -o ${meta.id}_sorted.cram ${bam} samtools index -@ ${task.cpus} ${meta.id}_sorted.cram samtools quickcheck -v ${meta.id}_sorted.cram samtools flagstat -@ ${task.cpus} ${bam} > bam.flagstat @@ -66,7 +67,20 @@ process CRAM_ARCHIVE { echo "ERROR: the CRAM of ${meta.id} does not hold the same reads as its BAM (flagstat above: < BAM, > CRAM)" >&2 exit 1 fi - echo "${meta.id}: BAM \$(du -k ${bam} | cut -f1) kB, CRAM \$(du -k ${meta.id}_sorted.cram | cut -f1) kB, flagstat equal" + # Match every record field and tag, independent of read/tag/group order. + # MD/NM are stored explicitly and never regenerated while checking. + # pipefail must propagate decoder failures through tail and sort. + set -o pipefail + checksum_rows() { + samtools checksum -@ ${task.cpus} -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "\$@" | tail -n +2 | LC_ALL=C sort + } + checksum_rows ${bam} > bam.checksum + checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${meta.id}_sorted.cram > ${meta.id}_sorted.cram.checksum + if ! diff bam.checksum ${meta.id}_sorted.cram.checksum >&2; then + echo "ERROR: the CRAM content of ${meta.id} differs from its BAM" >&2 + exit 1 + fi + echo "${meta.id}: BAM \$(du -k ${bam} | cut -f1) kB, CRAM \$(du -k ${meta.id}_sorted.cram | cut -f1) kB, flagstat and content checksums equal" cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -76,7 +90,7 @@ process CRAM_ARCHIVE { stub: """ - touch ${meta.id}_sorted.cram ${meta.id}_sorted.cram.crai ${meta.id}_sorted.cram.flagstat + touch ${meta.id}_sorted.cram ${meta.id}_sorted.cram.crai ${meta.id}_sorted.cram.flagstat ${meta.id}_sorted.cram.checksum cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -103,7 +117,7 @@ process CRAM_TO_BAM { script: """ - samtools view -@ ${task.cpus} -b --reference ${reference} -o ${meta.id}_from_cram.bam ${cram} + samtools view -@ ${task.cpus} -b --reference ${reference} --input-fmt-option decode_md=0 -o ${meta.id}_from_cram.bam ${cram} samtools index -@ ${task.cpus} ${meta.id}_from_cram.bam samtools quickcheck -v ${meta.id}_from_cram.bam samtools flagstat -@ ${task.cpus} --input-fmt-option reference=${reference} ${cram} > cram.flagstat @@ -113,6 +127,20 @@ process CRAM_TO_BAM { exit 1 fi + # Match every record field and tag, independent of read/tag/group order. + # MD/NM are stored explicitly and never regenerated while checking. + # pipefail must propagate decoder failures through tail and sort. + set -o pipefail + checksum_rows() { + samtools checksum -@ ${task.cpus} -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "\$@" | tail -n +2 | LC_ALL=C sort + } + checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${cram} > cram.checksum + checksum_rows ${meta.id}_from_cram.bam > bam.checksum + if ! diff cram.checksum bam.checksum >&2; then + echo "ERROR: the restored BAM content differs from ${cram}" >&2 + exit 1 + fi + cat <<-END_VERSIONS > versions.yml "${task.process}": samtools: ${task.container.replaceFirst(/^[^:@]+[:@]/, '')} diff --git a/scripts/34-cram-archive.sh b/scripts/34-cram-archive.sh index f3144ed30..0b50a0e3b 100755 --- a/scripts/34-cram-archive.sh +++ b/scripts/34-cram-archive.sh @@ -11,8 +11,8 @@ # A CRAM stores each read as its difference from the reference, so it can only # be read with the same reference FASTA it was written with: keep that file # (REF_FASTA) as long as you keep the CRAM. The check is that the CRAM passes -# `samtools quickcheck` and that `samtools flagstat` of the CRAM and of the BAM -# agree line for line (the same number of reads, mapped, paired, duplicates). +# `samtools quickcheck`, matching `flagstat` counts and matching full-content +# `samtools checksum` reports (all records, fields and auxiliary tags). # The BAM is deleted only with --delete-bam, only after that check passed in # this run, and never when it failed. --restore writes the BAM back the same # way, checked against the CRAM, for the bash steps, which read the BAM (the @@ -89,6 +89,27 @@ same_reads() { fi } +# The checksum reads every block, unlike flagstat. Use all record fields and +# tags, including secondary/supplementary records, with order-independent +# aggregation. Normalize the format-level repairs CRAM makes to unmapped +# records and CIGARs. Keep explicit MD/NM in the encoder and disable their +# generation on decode, so missing tags stay missing. Only the filename +# header is removed; sorting makes read-group row order immaterial too. +checksum_rows() { + local -a ref=() + case "$1" in *.cram) ref=(--input-fmt-option "reference=${REF_FASTA_C}" --input-fmt-option decode_md=0) ;; esac + run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ + samtools checksum -@ "$THREADS" -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' \ + ${ref[@]+"${ref[@]}"} "$1" | tail -n +2 | LC_ALL=C sort +} +content_checksum() { atomic_out "$2" checksum_rows "$1"; } +same_content() { + if [ ! -s "$1" ] || [ ! -s "$2" ] || ! diff "$1" "$2" >&2; then + echo " content checksum differs or is missing" >&2 + return 1 + fi +} + if [ "$MODE" = --restore ]; then for f in "$CRAM" "${CRAM}.crai"; do [ -f "$f" ] || { echo "ERROR: File not found: ${f}" >&2; exit 1; } @@ -100,13 +121,13 @@ if [ "$MODE" = --restore ]; then echo "Writing ${BAM} from ${CRAM}..." # fail_restore WHY: remove the partial BAM and its index, then stop. fail_restore() { - rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" + rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" "${ALN}/${SAMPLE}_sorted.part.bam.checksum" echo "ERROR: $1 Nothing was kept." >&2 exit 1 } rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" run_in --cpus "$THREADS" --memory 4g "$SAMTOOLS_IMAGE" \ - samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" \ + samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" --input-fmt-option decode_md=0 \ -o "${C}.part.bam" "${C}.cram" || fail_restore "samtools could not write the BAM from the CRAM." run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ samtools index -@ "$THREADS" "${C}.part.bam" || fail_restore "samtools could not index the BAM." @@ -115,6 +136,11 @@ if [ "$MODE" = --restore ]; then run_in "$SAMTOOLS_IMAGE" samtools quickcheck -v "${C}.part.bam" || fail_restore "The BAM fails samtools quickcheck." same_reads "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" "${ALN}/${SAMPLE}_sorted.cram.flagstat" \ || fail_restore "The BAM written from the CRAM does not hold the same reads." + content_checksum "${C}.cram" "${CRAM}.checksum" || fail_restore "samtools checksum failed on the CRAM." + content_checksum "${C}.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.checksum" || fail_restore "samtools checksum failed on the BAM." + same_content "${ALN}/${SAMPLE}_sorted.part.bam.checksum" "${CRAM}.checksum" \ + || fail_restore "The restored BAM content differs from the CRAM." + mv -f "${ALN}/${SAMPLE}_sorted.part.bam.checksum" "${BAM}.checksum" mv -f "${ALN}/${SAMPLE}_sorted.part.bam.bai" "${BAM}.bai" mv -f "${ALN}/${SAMPLE}_sorted.part.bam" "$BAM" rm -f "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" @@ -133,14 +159,15 @@ done # trusted: the BAM may have been aligned again since. PART="${ALN}/${SAMPLE}_sorted.part.cram" discard() { - rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" + rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" "${ALN}/${SAMPLE}_sorted.part.cram.checksum" echo "ERROR: $1 The CRAM was removed; the BAM is kept." >&2 exit 1 } -rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" +rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" "${ALN}/${SAMPLE}_sorted.part.cram.checksum" echo "Writing ${CRAM}..." run_in --cpus "$THREADS" --memory 4g "$SAMTOOLS_IMAGE" \ samtools view -@ "$THREADS" -C --reference "$REF_FASTA_C" \ + --output-fmt-option store_md=1 --output-fmt-option store_nm=1 \ -o "${C}.part.cram" "${C}.bam" || discard "samtools could not write the CRAM." echo "Checking the CRAM against the BAM..." @@ -152,6 +179,11 @@ flagstat "${C}.bam" "${ALN}/${SAMPLE}_sorted.bam.flagstat" || discard "samtools flagstat "${C}.part.cram" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" || discard "samtools flagstat failed on the CRAM." same_reads "${ALN}/${SAMPLE}_sorted.bam.flagstat" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" \ || discard "The CRAM does not hold the same reads as the BAM." +content_checksum "${C}.bam" "${BAM}.checksum" || discard "samtools checksum failed on the BAM." +content_checksum "${C}.part.cram" "${PART}.checksum" || discard "samtools checksum failed on the CRAM." +same_content "${BAM}.checksum" "${PART}.checksum" \ + || discard "The CRAM content differs from the BAM." +mv -f "${PART}.checksum" "${CRAM}.checksum" mv -f "${PART}.crai" "${CRAM}.crai" mv -f "$PART" "$CRAM" mv -f "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" "${ALN}/${SAMPLE}_sorted.cram.flagstat" diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 7f5b093d3..9ea0dd95d 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -180,3 +180,4 @@ y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh +qc-3-cram-checksum modules/local/cram_archive/* scripts/34-cram-archive.sh scripts/lib/common.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 5c80940c7..066ae31fb 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -179,3 +179,4 @@ y-str-1 shard 42 - y-tree-current-1-placement shard 21 nextflow-hardening-2-nf rust-z2e-readers shard 1 - rust-c2b-traits shard 2 - +qc-3-cram-checksum shard 40 - diff --git a/tests/e2e/qc-3-cram-checksum.sh b/tests/e2e/qc-3-cram-checksum.sh new file mode 100755 index 000000000..e1df0c207 --- /dev/null +++ b/tests/e2e/qc-3-cram-checksum.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Synthetic read content, including secondary/supplementary reads and tags. +# A quality, sequence or custom-tag change leaves flagstat unchanged. Each +# must stop --delete-bam and restore before a partial output is published. +. "$(dirname "$0")/lib.sh" +export GENOME_DIR="${CASE_TMP}/genome" +export REF_FASTA="${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" +C=checksum +A="${GENOME_DIR}/${C}/aligned" +rm -rf "$A" +mkdir -p "${GENOME_DIR}/reference" "$A" "${CASE_TMP}/fault" +printf '>chr1\n%s\n' "$(printf 'ACGT%.0s' {1..100})" > "$REF_FASTA" +# Include correct and incorrect MD/NM, missing MD/NM, a custom tag, a reverse +# read, a secondary read, a supplementary read, and placed/unplaced unmapped +# reads. Unmapped MAPQ is normalized by samtools' checksum sanitizer. +printf '@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr1\tLN:400\n@RG\tID:rg1\tSM:synthetic\n' > "${A}/in.sam" +for flag in 0 16 256 2048; do + printf 'read%s\t%s\tchr1\t1\t60\t4M\t*\t0\t0\tACGT\tIIII\tMD:Z:4\tNM:i:0\tXZ:Z:kept\tRG:Z:rg1\n' "$flag" "$flag" +done >> "${A}/in.sam" +printf 'wrong-md\t0\tchr1\t5\t60\t4M\t*\t0\t0\tACGT\tIIII\tMD:Z:0A3\tNM:i:1\nunmapped-placed\t4\tchr1\t9\t30\t*\t*\t0\t0\tACGT\tIIII\nunmapped\t4\t*\t0\t0\t*\t*\t0\t0\tACGT\tIIII\n' >> "${A}/in.sam" +sam faidx reference/GRCh38_no_alt_analysis_set.fasta +sam view -b -o "${C}/aligned/original.bam" "${C}/aligned/in.sam" +sam index "${C}/aligned/original.bam" + +# After a real conversion, rewrite its output with one changed field. Both +# commands succeed and all read flags/counts stay the same. For restore the +# same wrapper changes the partial BAM instead. +cat > "${CASE_TMP}/fault/docker" <<'WRAP' +#!/usr/bin/env bash +set -euo pipefail +self_dir="$(cd "$(dirname "$0")" && pwd)" +IFS=: read -r -a dirs <<< "$PATH" +keep=(); real="" +for d in "${dirs[@]}"; do + [ "$d" = "$self_dir" ] && continue + keep+=("$d") + if [ -z "$real" ] && [ -x "${d}/docker" ]; then real="${d}/docker"; fi +done +PATH=$(IFS=:; echo "${keep[*]}"); export PATH +[ -n "$real" ] || exit 127 +"$real" "$@" +args=" $* " +case "$args" in + *" samtools view "*" -o "*".part.${FAULT_FORMAT} "*) ;; + *) exit 0 ;; +esac +out=$(grep -oE "/genome/[^ ]+\.part\.${FAULT_FORMAT}" <<< "$args" | head -n 1) +"$real" run --rm -i -u "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" "$SAMTOOLS_IMAGE" \ + bash -euo pipefail -c ' + out=$1; fault=$2; ref=/genome/reference/GRCh38_no_alt_analysis_set.fasta + samtools view -T "$ref" --input-fmt-option decode_md=0 "$out" | + awk -F "\t" -v OFS="\t" -v fault="$fault" '\'' + fault == "reorder" && NR <= 4 { + # All four share one coordinate, so reversing them stays sorted. + t=$NF; $NF=$(NF-1); $(NF-1)=t + rows[NR]=$0 + if (NR == 4) for (j=4; j>=1; j--) print rows[j] + next + } + NR == 1 && fault == "quality" { $11 = "H" substr($11, 2) } + NR == 3 && fault == "sequence" { $10 = "T" substr($10, 2) } + NR == 4 && fault == "tag" { for (i=12; i<=NF; i++) if ($i ~ /^XZ:/) $i="XZ:Z:changed" } + { print } + '\'' > /tmp/changed.sam + samtools view -H "$out" > /tmp/changed-header.sam + cat /tmp/changed-header.sam /tmp/changed.sam > /tmp/changed-all.sam + if [[ $out == *.cram ]]; then + samtools view -C -T "$ref" --output-fmt-option store_md=1 --output-fmt-option store_nm=1 -o "$out" /tmp/changed-all.sam + else + samtools view -b -o "$out" /tmp/changed-all.sam + fi + echo "fault: $fault in $out" >&2 + ' bash "$out" "$FAULT" +WRAP +chmod +x "${CASE_TMP}/fault/docker" +export SAMTOOLS_IMAGE + +seed_bam() { + cp -f "${A}/original.bam" "${A}/${C}_sorted.bam" + cp -f "${A}/original.bam.bai" "${A}/${C}_sorted.bam.bai" +} +for fault in quality sequence tag; do + seed_bam + FAULT=$fault FAULT_FORMAT=cram PATH="${CASE_TMP}/fault:${PATH}" run_step 34-cram-archive.sh "$C" --delete-bam + check "${fault}: fault injected" has "^fault: ${fault} " "$(cat "$STEP_LOG")" + check "${fault}: archive refuses changed content" test "$STEP_RC" -ne 0 + check "${fault}: BAM is unchanged" cmp -s "${A}/original.bam" "${A}/${C}_sorted.bam" + check "${fault}: BAM index stays" test -s "${A}/${C}_sorted.bam.bai" + check "${fault}: no partial CRAM" test ! -e "${A}/${C}_sorted.part.cram" + check "${fault}: no published CRAM" test ! -e "${A}/${C}_sorted.cram" +done +seed_bam +FAULT=reorder FAULT_FORMAT=cram PATH="${CASE_TMP}/fault:${PATH}" run_step 34-cram-archive.sh "$C" --delete-bam +check "reordered read/tag control was injected" has "^fault: reorder " "$(cat "$STEP_LOG")" +check_step_exit 34-cram-archive.sh +check "faithful archive deletes BAM" test ! -e "${A}/${C}_sorted.bam" +check "archive keeps checksum evidence" test -s "${A}/${C}_sorted.cram.checksum" +for fault in quality sequence tag; do + rm -f "${A}/${C}_sorted.bam" "${A}/${C}_sorted.bam.bai" + FAULT=$fault FAULT_FORMAT=bam PATH="${CASE_TMP}/fault:${PATH}" run_step 34-cram-archive.sh "$C" --restore + check "${fault}: restore fault injected" has "^fault: ${fault} " "$(cat "$STEP_LOG")" + check "${fault}: restore refuses changed content" test "$STEP_RC" -ne 0 + check "${fault}: restore leaves no BAM" test ! -e "${A}/${C}_sorted.bam" + check "${fault}: restore leaves no partial BAM" test ! -e "${A}/${C}_sorted.part.bam" +done +rm -f "${A}/${C}_sorted.bam" "${A}/${C}_sorted.bam.bai" +run_step 34-cram-archive.sh "$C" --restore +check_step_exit 34-cram-archive.sh +check "faithful restore writes BAM and index" test -s "${A}/${C}_sorted.bam" -a -s "${A}/${C}_sorted.bam.bai" +# No generated MD/NM tags are added and all explicit tags survive, independent +# of tag order. checksum sanitizes only format-level normalization. +sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/original.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/original.checksum" +sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/${C}_sorted.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/restored.checksum" +check "round trip preserves read content and explicit tags" cmp -s "${CASE_TMP}/original.checksum" "${CASE_TMP}/restored.checksum" +finish diff --git a/tests/fake-docker/qc-steps-33-34.sh b/tests/fake-docker/qc-steps-33-34.sh index fdcf53e2c..4350e5ca4 100755 --- a/tests/fake-docker/qc-steps-33-34.sh +++ b/tests/fake-docker/qc-steps-33-34.sh @@ -12,7 +12,7 @@ # reruns VerifyBamID2 with --DisableSanityCheck and records it; any other # VerifyBamID2 failure, or missing data, stops the step. # Step 34: a CRAM whose flagstat differs from the BAM's, or that fails -# quickcheck, is removed and the BAM kept, also with --delete-bam; the BAM is +# quickcheck or its content checksum fails, is removed and the BAM kept; the BAM is # deleted only after the check passed; --restore refuses to write over a BAM # and leaves no partial BAM when it fails; archive and restore both stop # while another run holds the sample's lock, and the lock goes with its run. @@ -86,6 +86,15 @@ case "$*" in "samtools quickcheck"*) for a in "$@"; do last=$a; done if [ "${FAKE_QUICKCHECK_FAIL:-}" = "${last##*.}" ]; then echo "${last} had no EOF block" >&2; exit 1; fi ;; + "samtools checksum"*) + for a in "$@"; do last=$a; done + case "${FAKE_CHECKSUM_FAIL:-}:$last" in + cram:*.cram|bam:*.bam) echo "checksum: decode failed" >&2; exit 1 ;; + esac + [ "${FAKE_CHECKSUM_EMPTY:-}" != "${last##*.}" ] || exit 0 + sum=good + case "$last" in *.cram) sum=${FAKE_CRAM_CONTENT:-good} ;; esac + printf '# Checksum for file: %s\nall all 1000 %s\n' "$last" "$sum" ;; "samtools flagstat"*) for a in "$@"; do last=$a; done n=1000 @@ -173,6 +182,17 @@ FAKE_QUICKCHECK_FAIL=cram run_expect 1 cram-truncated "${SCRIPTS}/34-cram-archiv output_has cram-truncated 'fails samtools quickcheck' [ -s "${A}/sample1_sorted.bam" ] || fail "the BAM was deleted although the CRAM failed quickcheck" +FAKE_CRAM_CONTENT=changed run_expect 1 cram-content "${SCRIPTS}/34-cram-archive.sh" sample1 --delete-bam +output_has cram-content 'CRAM content differs from the BAM' +[ -s "${A}/sample1_sorted.bam" ] && [ -e "${A}/sample1_sorted.bam.bai" ] || fail "a content mismatch deleted the BAM or index" +[ ! -e "${A}/sample1_sorted.part.cram" ] && [ ! -e "${A}/sample1_sorted.part.cram.checksum" ] || fail "a content mismatch left partial files" +for format in bam cram; do + FAKE_CHECKSUM_FAIL=$format run_expect 1 "cram-checksum-fails-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --delete-bam + output_has "cram-checksum-fails-$format" 'samtools checksum failed' + FAKE_CHECKSUM_EMPTY=$format run_expect 1 "cram-checksum-empty-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --delete-bam + [ -s "${A}/sample1_sorted.bam" ] || fail "an empty or failed checksum deleted the BAM" +done + run_expect 0 cram-keep "${SCRIPTS}/34-cram-archive.sh" sample1 [ -s "${A}/sample1_sorted.cram" ] && [ -e "${A}/sample1_sorted.cram.crai" ] || fail "no checked CRAM" [ -s "${A}/sample1_sorted.bam" ] || fail "the BAM was deleted without --delete-bam" @@ -214,6 +234,15 @@ rm -f "${A}/sample1_sorted.bam" "${A}/sample1_sorted.bam.bai" FAKE_CRAM_READS=999 run_expect 1 cram-restore-differs "${SCRIPTS}/34-cram-archive.sh" sample1 --restore [ ! -e "${A}/sample1_sorted.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam" ] || fail "--restore kept a BAM that does not match the CRAM" +FAKE_CRAM_CONTENT=changed run_expect 1 cram-restore-content "${SCRIPTS}/34-cram-archive.sh" sample1 --restore +output_has cram-restore-content 'restored BAM content differs from the CRAM' +[ ! -e "${A}/sample1_sorted.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam.checksum" ] || fail "a restore content mismatch left a BAM or partial checksum" +for format in bam cram; do + FAKE_CHECKSUM_FAIL=$format run_expect 1 "restore-checksum-fails-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --restore + FAKE_CHECKSUM_EMPTY=$format run_expect 1 "restore-checksum-empty-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --restore + [ ! -e "${A}/sample1_sorted.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam" ] || fail "a restore with no checksum left a BAM" +done + FAKE_VIEW_FAIL=1 run_expect 1 cram-restore-write-fails "${SCRIPTS}/34-cram-archive.sh" sample1 --restore output_has cram-restore-write-fails 'samtools could not write the BAM from the CRAM' [ ! -e "${A}/sample1_sorted.part.bam" ] || fail "a failed --restore left its partial BAM" From ba002b007be67ae7e303adc3ac2d01f612c835f3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:16:02 +0100 Subject: [PATCH 47/86] feat(live): open the run reads inline from the genome map Bead: pgp-iqon.11 --- CHANGELOG.md | 2 + assets/live/map.css | 24 ++++++ assets/live/map.js | 141 ++++++++++++++++++++++++++++++++--- assets/viewer/serve.py | 2 + assets/viewer/viewer.js | 5 ++ bin/owngenome_live.py | 6 +- bin/owngenome_live_viewer.py | 2 +- docs/viewer.md | 8 +- 8 files changed, 176 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..9382a20d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer; Esc closes it and returns focus to the map. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/assets/live/map.css b/assets/live/map.css index 67cf5fe34..ad413123e 100644 --- a/assets/live/map.css +++ b/assets/live/map.css @@ -69,6 +69,30 @@ .gm-zoom { display: inline-flex; border: 1px solid var(--hair-2); border-radius: 999px; overflow: hidden; } .gm-zoom[hidden] { display: none; } .gm-zoom button { border: 0; border-radius: 0; background: transparent; color: var(--ink-2); padding: 6px 14px; font-size: 14px; line-height: 1; } +/* The reads take the table's place, while the map keeps its camera above them. */ +.gm-reads-pill { position: absolute; top: 0; right: 8px; border: 1px solid var(--brass-dim); border-radius: 999px; + color: var(--brass); background: var(--page); padding: 6px 12px; font-size: 12px; } +.gm-reads-hint { color: var(--brass); font-size: 12px; margin: 6px 0; } +.gm-reads { box-sizing: border-box; height: 330px; display: flex; flex-direction: column; min-width: 0; + margin-top: 10px; border: 1px solid var(--brass-dim); border-radius: 8px; overflow: hidden; background: var(--paper); } +.gm-reads[hidden], .gm-reads-pill[hidden], .gm-reads-hint[hidden] { display: none; } +.gm-reads-head { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 12px; padding: 8px 10px; font-size: 12px; } +.gm-reads-head b { font: 400 19px var(--serif); } +.gm-reads-head button { padding: 4px 8px; font-size: 12px; } +.gm-reads-place { overflow-wrap: anywhere; color: var(--brass); } +.gm-reads-who { margin: 0; padding: 0 10px 6px; font-size: 12px; color: var(--ink-2); } +.gm-reads-body { flex: 1; min-height: 0; padding: 0 8px; font-size: 13px; overflow: auto; } +.gm-reads-frame { display: block; width: 100%; height: 100%; border: 0; background: #fff; } +@media (prefers-reduced-motion: no-preference) { + .gm-reads { animation: gm-reads-in 180ms ease-out; } + @keyframes gm-reads-in { from { transform: translateY(20px); opacity: 0; } to { transform: translateY(0); opacity: 1; } } +} +@media (max-width: 520px) { + .gm-reads { height: 300px; } + .gm-gauge { max-width: 100%; overflow-x: auto; } + .gm-reads-head { gap: 4px 8px; padding: 6px; } + .gm-reads-head b { font-size: 16px; } +} /* Search (/): one box over gene names, band names, the findings drawn and a typed place. */ .gm-find { position: relative; max-width: 420px; } .gm-find-box { width: 100%; box-sizing: border-box; background: var(--paper); color: var(--ink); border: 1px solid var(--hair-2); diff --git a/assets/live/map.js b/assets/live/map.js index d207f45ec..0af553b75 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -148,7 +148,8 @@ var stopC = el('button', { type: 'button', 'data-stop': 'chromosome' }, 'Chromosome'); var stopR = el('button', { type: 'button', 'data-stop': 'region' }, 'Region'); var stopN = el('button', { type: 'button', 'data-stop': 'gene' }, 'Gene'); - add(gauge, stopG, stopC, stopR, stopN); + var stopReads = el('button', { type: 'button', 'data-stop': 'reads' }, 'Reads'); + add(gauge, stopG, stopC, stopR, stopN, stopReads); // Zoom buttons for an open chromosome: the touch path to what the wheel and the + and - keys do. var zoom = el('div', { 'class': 'gm-zoom', role: 'group', 'aria-label': 'Zoom', hidden: 'hidden' }); var zoomOut = el('button', { type: 'button', 'aria-label': 'Zoom out', 'data-zoom': 'out' }, '−'); @@ -168,14 +169,23 @@ var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); var toast = el('div', { 'class': 'gm-toast', role: 'status', hidden: 'hidden' }); - add(stage, canvas, tip, toast); + var readsPill = el('button', { type: 'button', 'class': 'gm-reads-pill', hidden: 'hidden', 'aria-controls': 'gm-reads', 'aria-expanded': 'false' }, 'See the reads · R'); + var readsHint = el('p', { 'class': 'gm-reads-hint', hidden: 'hidden' }, 'Under 50 kb: the reads are one key away · R'); + add(stage, canvas, tip, toast, readsPill); + var reads = el('section', { 'class': 'gm-reads', id: 'gm-reads', 'aria-label': 'Reads', hidden: 'hidden' }); + var readsPlace = el('span', { 'class': 'mono gm-reads-place' }); + var readsFull = el('a', { target: '_blank', rel: 'noopener' }, 'Open full viewer ↗'); + var readsClose = button('Back to the map · Esc', closeReads, 'gm-reads-close'); + var readsWho = el('p', { 'class': 'gm-reads-who' }); + var readsBody = el('div', { 'class': 'gm-reads-body' }); + add(reads, add(el('div', { 'class': 'gm-reads-head' }), el('b', {}, 'Reads'), readsPlace, readsFull, readsClose), readsWho, readsBody); // The marks drawn, as links: the keyboard and screen-reader path to each finding (Tab steps through them). var marks = el('ul', { 'class': 'gm-marks', 'aria-label': 'Findings on the map' }); var fallback = el('div', { 'class': 'gm-fallback', hidden: 'hidden' }); // The whole-genome strip over an open chromosome, and the In view table under the map (strip(), table()). var strip = el('canvas', { 'class': 'gm-strip', role: 'img', hidden: 'hidden' }); var inview = el('details', { 'class': 'gm-inview' }); - add(main, head, strip, stage, inview, marks, fallback); + add(main, head, strip, stage, readsHint, reads, inview, marks, fallback); var margin = el('aside', { 'class': 'gm-margin' }); var mkick = el('div', { 'class': 'fig-label' }, 'The whole genome'); var mline = el('div', { 'class': 'gm-mline mono' }); @@ -206,6 +216,7 @@ var hatch = null; // The finding whose card the margin shows (an id of drawn() only), and the mark the keyboard is on. var sel = null, focusId = null; + var readsFrame = null, readsId = null, readsView = '', readsLocus = ''; // The gene files, fetched once each: per chromosome when a view first reaches the region level, the // names when the search box gets its first key. Reference data, the same for every reader. var genes = {}, names = null; @@ -559,7 +570,11 @@ ctx.beginPath(); ctx.moveTo(wa, L.over + 12); ctx.lineTo(g, L.ruler); ctx.moveTo(wz, L.over + 12); ctx.lineTo(g + W, L.ruler); ctx.stroke(); ctx.globalAlpha = k; // The ruler, in Mb. - ctx.strokeStyle = 'rgba(255,255,255,.14)'; + if (level() === 'reads') { + ctx.fillStyle = 'rgba(200,171,120,.12)'; + ctx.fillRect(g, L.ruler - 2, W, 23); + } + ctx.strokeStyle = level() === 'reads' ? BRASS : 'rgba(255,255,255,.14)'; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(g, L.ruler); ctx.lineTo(g + W, L.ruler); ctx.stroke(); var stp = niceStep(v[2], W); @@ -1011,7 +1026,10 @@ stopG.setAttribute('aria-pressed', String(lv === 'genome')); stopC.setAttribute('aria-pressed', String(lv === 'chromosome')); stopR.setAttribute('aria-pressed', String(lv === 'region')); - stopN.setAttribute('aria-pressed', String(close_(lv))); + stopN.setAttribute('aria-pressed', String(lv === 'gene')); + stopReads.setAttribute('aria-pressed', String(lv === 'reads')); + readsPill.hidden = readsHint.hidden = lv !== 'reads'; + syncReads(); zoom.hidden = lv === 'genome'; crumb.textContent = ''; crumb.hidden = lv === 'genome'; // the gauge already says Genome @@ -1190,9 +1208,101 @@ add(card, facts); if (r.phrase) { add(card, el('p', { 'class': 'gm-card-phrase' }, r.phrase)); } var foot = el('p', { 'class': 'gm-card-foot' }); - add(foot, el('a', { href: r.href }, 'The finding’s page ›'), ' ', button('Close the card', function () { pick(null); canvas.focus({ preventScroll: true }); })); + var rb = button('See the reads', function () { openReads(r); }, 'gm-card-reads'); + rb.setAttribute('data-id', r.id); + add(foot, el('a', { href: r.href }, 'The finding’s page ›'), ' ', rb, ' ', button('Close the card', function () { pick(null); canvas.focus({ preventScroll: true }); })); add(card, foot); } + + // --- reads: only the existing same-origin viewer loads tracks and byte ranges --- + // Every candidate comes from drawn(), after the page's own veil callbacks. No result body is fetched here. + function readsBase() { + var run = document.documentElement.dataset.run || M.id; + return '/runs/' + encodeURIComponent(run) + '/viewer/'; + } + function exactView() { + var v = view(), ch = chrom(); + return ch.n + ':' + Math.max(1, Math.round(v[0] + 1)) + '-' + Math.min(ch.len, Math.round(v[1])); + } + function findingLocus(r) { + var ch = M.chroms[M.by[r.chrom]], p = r.pos || Math.round((r.s + r.e) / 2) + 1; + return r.chrom + ':' + Math.max(1, p - 50) + '-' + Math.min(ch.len, p + 50); + } + function nearestInView() { + var v = view(), ch = chrom(), best = null, dist = Infinity; + drawn().forEach(function (r) { + if (r.chrom !== ch.n || r.e <= v[0] || r.s >= v[1]) { return; } + var d = Math.abs((r.s + r.e) / 2 - cam.c); + if (d < dist) { best = r; dist = d; } + }); + return best; + } + function setReadsLocus(locus) { + if (locus === readsLocus) { return; } + readsLocus = locus; + readsPlace.textContent = locus; + readsFull.href = readsBase() + '#' + locus; + if (readsFrame) { + // Changing only the fragment lets viewer.js search without reloading igv.js. + readsFrame.src = readsFull.href; + } + } + function closeReads() { + if (reads.hidden) { return; } + reads.hidden = true; + readsBody.textContent = ''; // stop reads requests, and discard a result that can close again + readsFrame = null; readsId = null; readsLocus = ''; readsView = ''; + readsPill.setAttribute('aria-expanded', 'false'); + inview.hidden = !!failure; + canvas.focus({ preventScroll: true }); + } + function syncReads() { + if (reads.hidden) { return; } + if (failure || (readsId && !rowOf(readsId))) { closeReads(); return; } + var v = exactView(); + if (v !== readsView) { readsView = v; setReadsLocus(v); } + } + function openReads(r) { + if (!M || failure || (r && !rowOf(r.id))) { return; } + if (!r && level() !== 'reads') { say('Zoom under 50 kb to see the reads, or open a finding’s card'); return; } + r = r || nearestInView(); + readsId = r ? r.id : null; + reads.hidden = false; + inview.hidden = true; + readsPill.setAttribute('aria-expanded', 'true'); + readsView = exactView(); + readsWho.textContent = M.label + ' · run ' + (document.documentElement.dataset.run || M.id) + + (r ? ' · ' + (r.gene || r.title) + (r.hgvs ? ' ' + r.hgvs : '') : ''); + var demo = document.documentElement.dataset.mode === 'replay'; + readsFull.hidden = demo; + if (demo) { + readsBody.textContent = 'The genome viewer is not in the demo'; + } else if (!readsFrame) { + readsBody.textContent = ''; + var frame = el('iframe', { title: 'Genome viewer reads', 'class': 'gm-reads-frame' }); + readsFrame = frame; + frame.addEventListener('load', function () { + if (readsFrame !== frame) { return; } + var doc = frame.contentDocument; + if (!doc) { return; } + // An unavailable viewer answers with its reason as plain text. Display that reason in the panel. + if (!doc.documentElement.hasAttribute('data-pgp-status')) { + readsBody.textContent = 'The viewer is off on this site: ' + doc.body.textContent.trim(); + readsFrame = null; + readsFull.hidden = true; + return; + } + doc.addEventListener('keydown', function (e) { + if (e.key === 'Escape') { e.preventDefault(); closeReads(); } + }); + }); + readsLocus = ''; // a new frame needs its initial URL even when reopening the same place + } + setReadsLocus(r ? findingLocus(r) : readsView); + if (readsFrame && !readsFrame.parentNode) { add(readsBody, readsFrame); } + readsClose.focus({ preventScroll: true }); + } + readsPill.addEventListener('click', function () { openReads(); }); // The finding nearest the view when none of this chromosome's is in it: the closest one on this // chromosome, else the first on the next chromosome that has one, in reading order. Null when one is in view. function nearest() { @@ -1520,7 +1630,7 @@ var open = level() !== 'genome', span = Math.exp(tgt.ls); var done = true; if (e.key === 'Escape') { - if (sel) { pick(null); } else if (open) { stepOut(); } else { done = false; } + if (!reads.hidden) { closeReads(); } else if (sel) { pick(null); } else if (open) { stepOut(); } else { done = false; } } else if (e.key === '0') { if (open) { closeChrom(true); } else { done = false; } } else if (e.key === '[' || e.key === ']') { var i = (open ? tgt.ci : (e.key === ']' ? -1 : M.chroms.length)) + (e.key === ']' ? 1 : -1); @@ -1548,7 +1658,10 @@ if (!M || failure || e.altKey || e.ctrlKey || e.metaKey || !fig.isConnected) { return; } var t = e.target, tag = t && t.tagName; if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || (t && t.isContentEditable)) { return; } - if (e.key === '/') { e.preventDefault(); findBox.focus(); } + if (e.defaultPrevented) { return; } + if (e.key === 'Escape' && !reads.hidden) { e.preventDefault(); closeReads(); } + else if (e.key === 'r' || e.key === 'R') { e.preventDefault(); openReads(); } + else if (e.key === '/') { e.preventDefault(); findBox.focus(); } else if (e.key === 't' || e.key === 'T') { e.preventDefault(); inview.open = !inview.open; } else if (e.key === 'l' || e.key === 'L') { e.preventDefault(); @@ -1581,6 +1694,13 @@ stopC.addEventListener('click', function () { stopTo('chromosome'); }); stopR.addEventListener('click', function () { stopTo('region'); }); stopN.addEventListener('click', function () { stopTo('gene'); }); + stopReads.addEventListener('click', function () { + stopTo('gene'); + openChrom(tgt.ci, tgt.c, 2000, true); + // The target is already at reads level; finish the camera motion before selecting its nearest row. + cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; + openReads(); + }); nearBtn.addEventListener('click', function () { goNearest(); }); window.addEventListener('hashchange', function () { readHash(false); }); window.addEventListener('popstate', function () { readHash(false); }); @@ -1943,6 +2063,8 @@ // --- loading --- function fail(why, words) { failure = why; + closeReads(); + readsHint.hidden = true; // The frame and the margin the last refresh scheduled would draw the old document again: drop them. // These lines and the failure guards of draw(), describe() and schedule() cover for each other, so no one // of them can go red alone: the failed-reload check undoes them together. @@ -2004,7 +2126,7 @@ failure = null; stage.hidden = false; marks.hidden = false; - inview.hidden = false; + inview.hidden = !reads.hidden; fallback.hidden = true; hatch = hatch || pattern(0.16); fig.setAttribute('data-pgp-map-parse-ms', String(Math.round(x.parse))); @@ -2043,6 +2165,7 @@ return; } if (!M) { return; } + syncReads(); cacheKey = ''; hideTip(); // a tooltip of a result closed since leaves the page drawCard(); // and so does the card of one diff --git a/assets/viewer/serve.py b/assets/viewer/serve.py index 435c9daeb..fc7d1624d 100755 --- a/assets/viewer/serve.py +++ b/assets/viewer/serve.py @@ -38,6 +38,8 @@ CSP = ("default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self' data: blob:; " "style-src 'self' 'unsafe-inline'; font-src 'self' data:; worker-src 'self' blob:; " "base-uri 'none'; form-action 'none'; frame-ancestors 'none'") +# Only the live site's viewer can be embedded by another page of that site. +LIVE_CSP = CSP.replace("frame-ancestors 'none'", "frame-ancestors 'self'") LOCUS_RE = re.compile(r"^([A-Za-z0-9._-]+)(?::([0-9][0-9,]*)(?:-([0-9][0-9,]*))?)?$") diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index 295234cc0..aadfcf506 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -43,6 +43,11 @@ return; } const cfg = await resp.json(); + cfg.tracks.forEach(function (track) { + if (track.type === "alignment") { + track.visibilityWindow = 50000; + } + }); document.getElementById("sample").textContent = cfg.sample; document.title = cfg.sample + " - Genome viewer"; const browser = await igv.createBrowser(document.getElementById("igv"), { diff --git a/bin/owngenome_live.py b/bin/owngenome_live.py index 1b3edbcf0..a712e43a4 100755 --- a/bin/owngenome_live.py +++ b/bin/owngenome_live.py @@ -78,7 +78,7 @@ FAIL_WINDOW_SECONDS = 15 * 60 LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} CSP = ("default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; img-src 'self' data:; " - "form-action 'self'; base-uri 'none'; frame-ancestors 'none'") + "form-action 'self'; base-uri 'none'; frame-src 'self'; frame-ancestors 'none'") # A run's own HTML (the report, CPSR, PharmCAT, MultiQC) carries inline styles and, for third-party # pages, scripts and outside links. It is sent sandboxed: no script runs, nothing is fetched from # outside, and it keeps its styles. allow-same-origin lets its links reach the run's other files @@ -915,12 +915,12 @@ def client(self): pass return peer - def headers_common(self, csp=CSP): + def headers_common(self, csp=CSP, frame_options="DENY"): self.send_header("Content-Security-Policy", csp) self.send_header("X-Content-Type-Options", "nosniff") self.send_header("Referrer-Policy", "no-referrer") self.send_header("Cache-Control", "no-store") - self.send_header("X-Frame-Options", "DENY") + self.send_header("X-Frame-Options", frame_options) def send_body(self, code, ctype, body, headers=(), csp=CSP): if isinstance(body, str): diff --git a/bin/owngenome_live_viewer.py b/bin/owngenome_live_viewer.py index 7edbf1bae..e7891bb49 100644 --- a/bin/owngenome_live_viewer.py +++ b/bin/owngenome_live_viewer.py @@ -108,7 +108,7 @@ def head(code, ctype, length, extra=()): req.send_header("Cross-Origin-Resource-Policy", "same-origin") for k, v in extra: req.send_header(k, v) - req.headers_common(serve.CSP) + req.headers_common(serve.LIVE_CSP, frame_options="SAMEORIGIN") req.end_headers() return head diff --git a/docs/viewer.md b/docs/viewer.md index 5c5170550..8b30532c8 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -80,7 +80,13 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the ## From the live site -`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` does it with the same page, the same Content-Security-Policy and the same files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF`, `SV_VCF`, `ALIGN_DIR` and `VCF_DIR` overrides do not apply there, and a track whose file or index is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. +`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` uses the same page and files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF`, `SV_VCF`, `ALIGN_DIR` and `VCF_DIR` overrides do not apply there, and a track whose file or index is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. + +On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. + +The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. + +The live viewer allows framing only by the same origin: it sends `frame-ancestors 'self'` and `X-Frame-Options: SAMEORIGIN`. Live pages allow `frame-src 'self'`. Other routes keep `frame-ancestors 'none'` and `X-Frame-Options: DENY`. The standalone local viewer keeps its original policy, which blocks all framing. This differs from the local viewer in one way that matters: the reads now cross the network, to whatever browser logged in. Only the byte ranges igv.js asks for travel, and only after the login, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS), or keep the site on loopback and forward it over ssh. Opening the viewer writes one row in the site's access log; the byte ranges write none. From 80137d4f8d947ed02b1b0beb0bd82e65ab639aa9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:16:23 +0100 Subject: [PATCH 48/86] test(recheck): red first for a failed lookup read as nothing newer, and a replayed shell string Adds the typed call reader (parse_call) without using it yet, and the tests that the next commit turns green: a lookup failure must not hide a newer ClinVar file here or print 'Nothing here is newer', and --rerun must refuse a command that is not a run-all.sh call. Bead: plat-25 Bead: plat-26 --- rust/crates/og-recheck/src/rerun.rs | 200 ++++++++++++++- rust/crates/og-recheck/src/tests.rs | 372 ++++++++++++++++++++++++++-- 2 files changed, 544 insertions(+), 28 deletions(-) diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index 26c27e091..78c9f242a 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -121,12 +121,206 @@ pub enum Rerun { Refused(u8, String), } +/// One shell word of a recorded command. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Token { + /// A word: its text, and how many of its first bytes stood unquoted. + Word(String, usize), + /// The `&&` between the `cd` and the call. + And, +} + +/// Bytes `printf %q` leaves unquoted anywhere in a word, and that mean +/// nothing to the shell there. `#` and `~` do too, except first in a word. +fn plain(b: u8, first: bool) -> bool { + b.is_ascii_alphanumeric() + || b"%+-./:=@_,".contains(&b) + || (!first && (b == b'#' || b == b'~')) + || b >= 0x80 +} + +/// The text of a `$'...'` quote from just after its `$'`; `None` for an +/// escape `printf %q` never writes. Moves `i` past the closing quote. +fn ansi_c(s: &[u8], i: &mut usize, out: &mut Vec) -> Option<()> { + loop { + let b = *s.get(*i)?; + *i += 1; + match b { + b'\'' => return Some(()), + b'\\' => { + let e = *s.get(*i)?; + *i += 1; + out.push(match e { + b'a' => 7, + b'b' => 8, + b'e' | b'E' => 27, + b'f' => 12, + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'v' => 11, + b'\\' | b'\'' | b'"' | b'?' => e, + b'0'..=b'7' => { + let mut v = u32::from(e - b'0'); + for _ in 0..2 { + match s.get(*i) { + Some(d @ b'0'..=b'7') => { + v = v * 8 + u32::from(d - b'0'); + *i += 1; + } + _ => break, + } + } + u8::try_from(v).ok()? + } + b'x' => { + let mut v = 0u32; + let mut n = 0; + while n < 2 + && let Some(d) = s.get(*i).and_then(|d| char::from(*d).to_digit(16)) + { + v = v * 16 + d; + *i += 1; + n += 1; + } + if n == 0 { + return None; + } + u8::try_from(v).ok()? + } + _ => return None, + }); + } + _ => out.push(b), + } + } +} + +/// The words of a command as `run-all.sh` writes it with `printf %q`: +/// plain bytes, `\x`, `'...'` and `$'...'`, words split on spaces and tabs, +/// and the `&&` operator. `None` for anything else a shell would act on (a +/// `;`, a pipe, a redirect, an expansion, a glob, a double quote, a line +/// break) and for text that is not UTF-8. +fn words(cmd: &str) -> Option> { + let s = cmd.as_bytes(); + let (mut out, mut i) = (Vec::new(), 0); + while i < s.len() { + if matches!(s[i], b' ' | b'\t') { + i += 1; + continue; + } + if s[i..].starts_with(b"&&") && s.get(i + 2).is_none_or(|b| matches!(*b, b' ' | b'\t')) { + out.push(Token::And); + i += 2; + continue; + } + let (mut w, mut lead, mut quoted) = (Vec::new(), 0, false); + while i < s.len() && !matches!(s[i], b' ' | b'\t') { + let b = s[i]; + if b == b'\\' { + let ch = cmd.get(i + 1..)?.chars().next()?; + if ch == '\n' { + return None; + } + let mut buf = [0u8; 4]; + w.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); + i += 1 + ch.len_utf8(); + quoted = true; + } else if b == b'\'' { + let end = s[i + 1..].iter().position(|&c| c == b'\'')? + i + 1; + w.extend_from_slice(&s[i + 1..end]); + i = end + 1; + quoted = true; + } else if b == b'$' && s.get(i + 1) == Some(&b'\'') { + i += 2; + ansi_c(s, &mut i, &mut w)?; + quoted = true; + } else if plain(b, w.is_empty() && !quoted) { + w.push(b); + i += 1; + if !quoted { + lead = w.len(); + } + } else { + return None; + } + } + out.push(Token::Word(String::from_utf8(w).ok()?, lead)); + } + Some(out) +} + +/// A recorded `run-all.sh` call as typed parts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Call { + /// The directory it was run from. + pub dir: String, + /// The switches it set, as `NAME`, value. + pub env: Vec<(String, String)>, + /// The `run-all.sh` it ran. + pub script: String, + /// Its arguments. + pub args: Vec, +} + +/// `meta command` as a typed call, or `None` when it is not the shape +/// `run-all.sh` writes: `cd DIR && [NAME=VALUE ...] .../scripts/run-all.sh +/// ARG ...`, every `NAME` one of `switches`. Nothing of it reaches a +/// shell. +pub fn parse_call(cmd: &str, switches: &str) -> Option { + let names: Vec<&str> = switches.split_whitespace().collect(); + let mut t = words(cmd)?.into_iter(); + let mut word = || match t.next() { + Some(Token::Word(w, lead)) => Some((w, lead)), + _ => None, + }; + let (cd, lead) = word()?; + if cd != "cd" || lead != 2 { + return None; + } + let (dir, _) = word()?; + if !matches!(t.next(), Some(Token::And)) { + return None; + } + let mut env = Vec::new(); + let script = loop { + let (w, lead) = match t.next() { + Some(Token::Word(w, lead)) => (w, lead), + _ => return None, + }; + match w.find('=') { + Some(eq) if eq < lead && names.contains(&&w[..eq]) => { + env.push((w[..eq].to_owned(), w[eq + 1..].to_owned())); + } + _ => break w, + } + }; + if !script.ends_with("/scripts/run-all.sh") { + return None; + } + let args = t + .map(|x| match x { + Token::Word(w, _) => Some(w), + Token::And => None, + }) + .collect::>>()?; + Some(Call { + dir, + env, + script, + args, + }) +} + /// Replays the last `run-all.sh` command of the sample (`meta command` of /// `logs/run_status.tsv`) with the same switches: the switch variables the /// run read (`meta switches`) are dropped from the environment, so the -/// command's own assignments decide. One `--rerun` per sample at a time: -/// `recheck/rerun.lock` is held from the running check through the end of -/// the replay. Records the rerun in `state` and the log. +/// command's own assignments decide. The command is read as a typed call +/// ([`parse_call`]) and `run-all.sh` is started by `bash` with its +/// arguments one by one; a command of any other shape is refused and +/// nothing runs. One `--rerun` per sample at a time: `recheck/rerun.lock` +/// is held from the running check through the end of the replay. Records +/// the rerun in `state` and the log. pub fn rerun( sample: &str, sample_dir: &Path, diff --git a/rust/crates/og-recheck/src/tests.rs b/rust/crates/og-recheck/src/tests.rs index 9d465286d..fb1fd6bea 100644 --- a/rust/crates/og-recheck/src/tests.rs +++ b/rust/crates/og-recheck/src/tests.rs @@ -4,7 +4,7 @@ use super::*; use crate::answers::{Offline, Online, PGS_API}; -use crate::rerun::{compare, launch_dir_command}; +use crate::rerun::{Call, compare, launch_dir_command, parse_call}; use og_fresh::http::GitHub; use og_fresh::pins::Pins; use std::fs; @@ -219,6 +219,27 @@ fn s(p: &Path) -> &str { p.to_str().unwrap() } +fn chmod_x(path: &Path) { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +/// A stand-in `scripts/run-all.sh` beside the genome folder that runs +/// `body`, recorded in `logs/run_status.tsv` the way `run-all.sh` records +/// itself: `cd` to the launch folder, then the script and `S1 male`. +fn stub_run_all(g: &Genome, body: &str) { + let launch = g.root.parent().unwrap().join("launch"); + let script = launch.join("scripts/run-all.sh"); + fs::create_dir_all(script.parent().unwrap()).unwrap(); + fs::write(&script, format!("#!/bin/sh\n{body}\n")).unwrap(); + chmod_x(&script); + let cmd = format!("cd '{}' && '{}' S1 male", s(&launch), s(&script)); + let st = fs::read_to_string(g.status()) + .unwrap() + .replace("echo replayed", &cmd); + fs::write(g.status(), st).unwrap(); +} + #[test] fn clinvar_newer_upstream_is_refresh_available_with_06_and_its_downstream() { let g = genome("cv", Opt::default()); @@ -568,20 +589,11 @@ fn after_rerun_the_check_is_taken_again_on_the_manifest_the_rerun_wrote() { .collect(); fs::write(&newman, text).unwrap(); let g = genome("after-g", Opt::default()); - let cmd = format!("cp '{}' '{}'", s(&newman), s(&g.manifest())); - let g = Genome { - root: { - let st = fs::read_to_string(g.status()) - .unwrap() - .replace("echo replayed", &cmd); - fs::write(g.status(), st).unwrap(); - gz( - &g.root.join("clinvar/clinvar_pathogenic_chr.vcf.gz"), - "##fileformat=VCFv4.1\n##fileDate=20261005\n#CHROM\n", - ); - g.root - }, - }; + stub_run_all(&g, &format!("cp '{}' '{}'", s(&newman), s(&g.manifest()))); + gz( + &g.root.join("clinvar/clinvar_pathogenic_chr.vcf.gz"), + "##fileformat=VCFv4.1\n##fileDate=20261005\n#CHROM\n", + ); let off = fix("offline.json"); let (rc, out, _) = cli(&[ "--sample", @@ -698,11 +710,7 @@ fn refresh_clinvar_holds_the_lock_while_setup_refreshes_and_lets_it_go_before_th .status() .unwrap(); let g = genome("refresh-held-g", Opt::default()); - let replay = probe(s(&g.root.join(CLINVAR_LOCK)), &seen, 20); - let st = fs::read_to_string(g.status()) - .unwrap() - .replace("echo replayed", &replay); - fs::write(g.status(), st).unwrap(); + stub_run_all(&g, &probe(s(&g.root.join(CLINVAR_LOCK)), &seen, 20)); let off = fix("offline.json"); let (rc, out, err) = cli(&[ "--sample", @@ -1049,11 +1057,7 @@ fn a_lookup_that_fails_after_the_replay_makes_the_command_exit_1() { text += "data\tpgs:PGS000099\tpgs_id=PGS000099;HmPOS_date=2022-07-29\n"; fs::write(&newman, text).unwrap(); let g = genome("after-fail-g", Opt::default()); - let cmd = format!("cp '{}' '{}'", s(&newman), s(&g.manifest())); - let st = fs::read_to_string(g.status()) - .unwrap() - .replace("echo replayed", &cmd); - fs::write(g.status(), st).unwrap(); + stub_run_all(&g, &format!("cp '{}' '{}'", s(&newman), s(&g.manifest()))); let off = fix("offline.json"); let (rc, out, _) = cli(&[ "--sample", @@ -1068,3 +1072,321 @@ fn a_lookup_that_fails_after_the_replay_makes_the_command_exit_1() { assert!(state.contains("pgs:PGS000099") && state.contains("\"class\": \"error\"")); assert_eq!(rc, 1, "{out}"); } + +/// Every lookup through the online client at a port where nothing listens: +/// a refused connection, not an HTTP error. +struct Refused(Online, u16); + +impl Answers for Refused { + fn clinvar(&self) -> og_fresh::Lookup> { + let url = format!("http://127.0.0.1:{}/clinvar", self.1); + og_fresh::http::Fetch::text(&self.0.client, &url).map(Some) + } + fn pinned(&self, key: &str) -> og_fresh::Lookup> { + let url = format!("http://127.0.0.1:{}/{key}", self.1); + og_fresh::http::Fetch::text(&self.0.client, &url).map(Some) + } + fn pgs(&self, id: &str) -> og_fresh::Lookup> { + self.0.pgs(id) + } + fn release(&self) -> og_fresh::Lookup { + self.0.release() + } + fn hosts(&self) -> Vec { + self.0.hosts() + } + fn offline(&self) -> bool { + false + } +} + +fn printed(c: &Check) -> String { + let mut o = Vec::new(); + print_check(c, &mut o).unwrap(); + String::from_utf8(o).unwrap() +} + +#[test] +fn a_failed_lookup_never_prints_nothing_newer_and_keeps_a_newer_file_here_rerun_ready() { + let g = genome( + "cv-down", + Opt { + installed_clinvar: Some("20261101"), + ..Opt::default() + }, + ); + let c = run_check( + &g, + &answers(&[( + "clinvar_file_date", + r#"{"error": "no answer (Could not resolve host)"}"#, + )]), + &this_pins(), + Some("3.66.0"), + ); + let cv = by_item(&c)["clinvar_file_date"]; + assert_eq!(cv.class, Class::RerunReady, "{cv:?}"); + assert!(cv.error.as_deref().unwrap().contains("resolve host")); + assert_eq!(c.rerun_ready_steps, CLINVAR_STEPS); + assert_eq!(c.errors(), 1); + let out = printed(&c); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!(out.contains("the lookup failed: no answer"), "{out}"); + assert!(out.contains("A rerun would redo steps 06,"), "{out}"); + assert!(out.contains("1 of 7 lookups failed"), "{out}"); +} + +#[test] +fn when_every_lookup_is_refused_the_check_says_it_could_not_check() { + let port = { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + l.local_addr().unwrap().port() + }; + let g = genome("all-refused", Opt::default()); + let r = Refused(online(port), port); + let c = run_check(&g, &r, &this_pins(), Some("3.66.0")); + for i in c + .items + .iter() + .filter(|i| i.error.is_some() || i.upstream.is_some()) + { + assert_eq!(i.class, Class::Error, "{i:?}"); + } + assert!(c.release.is_err()); + assert_eq!(c.errors(), 7); + assert!(c.affected_steps.is_empty()); + let out = printed(&c); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!( + out.contains("Could not check: all 7 lookups failed"), + "{out}" + ); + // The command: every answer an error, exit 1, the same closing line. + let p = tmp("all-err").join("all-err.json"); + fs::write( + &p, + r#"{"clinvar_file_date": {"error": "no answer"}, "VEP_CACHE_RELEASE": {"error": "no answer"}, + "PCGR_DATA_BUNDLE": {"error": "no answer"}, "PYPGX_BUNDLE_VERSION": {"error": "no answer"}, + "PHARMCAT_IMAGE": {"error": "no answer"}, "hla_database": {"error": "no answer"}, + "pgs:PGS000001": {"error": "no answer"}, "release": {"error": "no answer"}}"#, + ) + .unwrap(); + let (rc, out, _) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&p), + ]); + assert_eq!(rc, 1, "{out}"); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!( + out.contains("Could not check: all 7 lookups failed"), + "{out}" + ); + // Control: with every answer given, nothing newer is said. + let g = genome("all-answered", Opt::default()); + let c = run_check( + &g, + &answers(&[ + ("clinvar_file_date", r#"{"upstream": "20260901"}"#), + ("VEP_CACHE_RELEASE", r#"{"upstream": "116"}"#), + ]), + &this_pins(), + Some("3.66.0"), + ); + assert_eq!(c.errors(), 0); + let out = printed(&c); + assert!(out.contains("Nothing here is newer"), "{out}"); +} + +#[test] +fn a_recorded_command_is_read_as_a_typed_call() { + // As bash 5's printf %q writes them: a space, a comma, a tab, a quote, + // an empty word, a non-ASCII letter in a C locale. + let cmd = "cd /data/my\\ run && TOOLS=deepvariant\\,pharmcat NXF_VER=25.04.6 \ + /opt/og/scripts/run-all.sh S1 male --publish_dir out\\ dir/x\\;y it\\'s $'a\\tb' '' \ + $'\\303\\251' a~b a#b"; + let call = parse_call(cmd, "GENOME_DIR TOOLS NXF_VER").unwrap(); + assert_eq!( + call, + Call { + dir: "/data/my run".into(), + env: vec![ + ("TOOLS".into(), "deepvariant,pharmcat".into()), + ("NXF_VER".into(), "25.04.6".into()), + ], + script: "/opt/og/scripts/run-all.sh".into(), + args: [ + "S1", + "male", + "--publish_dir", + "out dir/x;y", + "it's", + "a\tb", + "", + "\u{e9}", + "a~b", + "a#b" + ] + .map(String::from) + .to_vec(), + } + ); + // Anything else a shell would act on is no call. + let base = "cd /d && /o/scripts/run-all.sh S1 male"; + for bad in [ + "touch /tmp/X && echo hi".to_owned(), + format!("{base}; rm -rf /tmp/X"), + format!("{base} ; rm -rf /tmp/X"), + format!("{base} && rm -rf /tmp/X"), + format!("{base} | tee x"), + format!("{base} > x"), + format!("{base} $(id)"), + format!("{base} `id`"), + format!("{base} \"quoted\""), + format!("{base} *"), + format!("{base} $HOME"), + format!("{base} #comment"), + format!("{base} ~/x"), + format!("{base} 'open"), + format!("{base} $'\\u00e9'"), + "cd /d && OTHER=1 /o/scripts/run-all.sh S1 male".to_owned(), + "cd /d && /o/scripts/evil.sh S1 male".to_owned(), + "cd /d /o/scripts/run-all.sh S1 male".to_owned(), + "\\cd /d && /o/scripts/run-all.sh S1 male".to_owned(), + "cd /d && 'TOOLS=x' /o/scripts/run-all.sh S1".to_owned(), + ] { + assert_eq!(parse_call(&bad, "TOOLS"), None, "{bad}"); + } +} + +#[test] +fn rerun_refuses_a_command_that_is_not_a_run_all_call_and_runs_nothing() { + let off = fix("offline.json"); + let base = tmp("inject"); + let victim = base.join("victim"); + let made = base.join("made"); + for (name, cmd) in [ + ("touch", format!("touch '{}' && echo not-run-all", s(&made))), + ( + "suffix", + format!( + "cd '{}' && '{}/scripts/run-all.sh' S1 male; rm -f '{}'", + s(&base), + s(&base), + s(&victim) + ), + ), + ] { + fs::write(&victim, "").unwrap(); + let g = genome(&format!("inject-{name}"), Opt::default()); + let st = fs::read_to_string(g.status()) + .unwrap() + .replace("echo replayed", &cmd); + fs::write(g.status(), st).unwrap(); + let (rc, out, err) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&off), + "--rerun", + ]); + assert_eq!(rc, 2, "{name}: {out}{err}"); + assert!(err.contains("not a run-all.sh call"), "{err}"); + assert!(!out.contains("Rerunning"), "{out}"); + assert!(!made.exists(), "{name}: the command ran"); + assert!(victim.exists(), "{name}: the suffix ran"); + let log = fs::read_to_string(g.sample().join("recheck/log.tsv")).unwrap(); + assert!(!log.contains("\trerun-start\t"), "{log}"); + } + // A call whose launch folder is gone is refused too. + let g = genome("inject-gone", Opt::default()); + let st = fs::read_to_string(g.status()).unwrap().replace( + "echo replayed", + &format!( + "cd '{}/gone' && '{}/scripts/run-all.sh' S1 male", + s(&base), + s(&base) + ), + ); + fs::write(g.status(), st).unwrap(); + let (rc, _, err) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&off), + "--rerun", + ]); + assert_eq!(rc, 2, "{err}"); + assert!(err.contains("is gone"), "{err}"); +} + +#[test] +fn rerun_starts_run_all_with_the_recorded_arguments_one_by_one() { + let g = genome("typed", Opt::default()); + let launch = g.root.parent().unwrap().join("launch dir"); + let script = launch.join("scripts/run-all.sh"); + fs::create_dir_all(script.parent().unwrap()).unwrap(); + let seen = g.root.parent().unwrap().join("seen.txt"); + fs::write( + &script, + format!( + "{{ printf '%s\\n' \"$PWD\" \"${{TOOLS-unset}}\" \"${{GENOME_DIR-unset}}\" \"$#\"; \ + for a in \"$@\"; do printf '[%s]\\n' \"$a\"; done; }} > '{}'\n", + s(&seen) + ), + ) + .unwrap(); + chmod_x(&script); + let q = |p: &Path| s(p).replace(' ', "\\ "); + let cmd = format!( + "cd {} && TOOLS=deepvariant\\,pharmcat {} S1 male --publish_dir out\\ dir it\\'s $'a\\tb' '' x\\;\\ touch\\ y", + q(&launch), + q(&script) + ); + let st = fs::read_to_string(g.status()) + .unwrap() + .replace("echo replayed", &cmd); + fs::write(g.status(), st).unwrap(); + let off = fix("offline.json"); + let (rc, out, err) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&off), + "--rerun", + ]); + assert_eq!(rc, 0, "{out}{err}"); + let got = fs::read_to_string(&seen).unwrap(); + let pwd = launch.canonicalize().unwrap(); + let first = got.lines().next().unwrap(); + assert!(Path::new(first).canonicalize().unwrap() == pwd, "{got}"); + let rest: Vec<&str> = got.lines().skip(1).collect(); + assert_eq!( + rest, + [ + "deepvariant,pharmcat", + // GENOME_DIR is a recorded switch the command does not set: dropped. + "unset", + "8", + "[S1]", + "[male]", + "[--publish_dir]", + "[out dir]", + "[it's]", + "[a\tb]", + "[]", + "[x; touch y]", + ], + "{got}" + ); + assert!(!launch.join("y").exists()); +} From ad9606554f06bee128f9da7bac642fefd0b28974 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:18:08 +0100 Subject: [PATCH 49/86] fix(freshness): verify clinical download links and metadata Bead: D11 --- rust/crates/og-fresh/src/lookups.rs | 74 ++++++++++ .../tests/screen_setup_clinical_default.rs | 135 +++++++++++++++--- versions.env | 2 + 3 files changed, 195 insertions(+), 16 deletions(-) diff --git a/rust/crates/og-fresh/src/lookups.rs b/rust/crates/og-fresh/src/lookups.rs index 70a6e37bb..14511e2a8 100644 --- a/rust/crates/og-fresh/src/lookups.rs +++ b/rust/crates/og-fresh/src/lookups.rs @@ -223,6 +223,77 @@ pub fn pcgr(c: &Ctx<'_>) -> Lookup { }) } +/// Verify a pinned cache URL and its entry in Ensembl's small CHECKSUMS file. +/// Large archives are only read with HEAD, never downloaded by freshness. +fn cache_download(c: &Ctx<'_>, pin: &str, label: &str) -> Lookup { + let release = c.pins.need(pin)?; + let url = vep_cache_template(c.common)?.replace("%s", release); + c.fetch.head(&url)?; + let (base, name) = url.rsplit_once('/').expect("cache URL has a basename"); + let checksums = c.fetch.text(&format!("{base}/CHECKSUMS"))?; + let listed = checksums.lines().any(|line| { + let fields: Vec<_> = line.split_whitespace().collect(); + fields.len() == 3 + && fields[2] == name + && fields[0].parse::().is_ok() + && fields[1].parse::().is_ok_and(|n| n > 0) + }); + if !listed { + return failed(format!("{name} has no usable entry in {base}/CHECKSUMS")); + } + Ok(Found { + row: row( + label, + release.into(), + "URL live; CHECKSUMS entry present".into(), + false, + String::new, + ), + upstream: Upstream::ReportOnly, + }) +} + +/// The release used by steps 13, 30, 23 and 31. +pub fn vep_download(c: &Ctx<'_>) -> Lookup { + cache_download(c, "VEP_CACHE_RELEASE", "VEP cache download") +} + +/// The separate release used inside CPSR's image. +pub fn cpsr_download(c: &Ctx<'_>) -> Lookup { + cache_download(c, "PCGR_VEP_CACHE_RELEASE", "CPSR cache download") +} + +/// The pinned PCGR bundle URL and size. SHA-256 is checked by setup, which +/// downloads the file; freshness deliberately reads only its headers. +pub fn pcgr_download(c: &Ctx<'_>) -> Lookup { + let bundle = c.pins.need("PCGR_DATA_BUNDLE")?; + let expected = c + .pins + .need("PCGR_DATA_BYTES")? + .parse::() + .map_err(|_| LookupError::Shape("PCGR_DATA_BYTES is not a byte count".into()))?; + let url = format!("https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.{bundle}.grch38.tgz"); + let headers = c.fetch.head(&url)?; + let size = headers + .get("Content-Length") + .and_then(|s| s.parse::().ok()); + if expected == 0 || size != Some(expected) { + return failed(format!( + "{url}: Content-Length {size:?}, expected {expected}" + )); + } + Ok(Found { + row: row( + "PCGR bundle download", + bundle.into(), + format!("URL live; {expected} bytes"), + false, + String::new, + ), + upstream: Upstream::ReportOnly, + }) +} + pub fn pypgx(c: &Ctx<'_>) -> Lookup { let image_tag = tag_of(c.pins, "PYPGX_IMAGE")?; let held = image_tag.split("--").next().unwrap_or("").to_owned(); @@ -667,6 +738,9 @@ pub type LookupFn = fn(&Ctx<'_>) -> Lookup; pub const DATABASES: &[(&str, LookupFn)] = &[ ("Ensembl / VEP cache", ensembl), ("PCGR / CPSR", pcgr), + ("VEP cache download", vep_download), + ("CPSR cache download", cpsr_download), + ("PCGR bundle download", pcgr_download), ("pypgx", pypgx), ("PharmCAT", pharmcat), ("IPD-IMGT/HLA", imgt), diff --git a/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs index 077cd2a27..0564a0ace 100644 --- a/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs +++ b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs @@ -15,39 +15,142 @@ struct Downloads { calls: Mutex>, } impl Fetch for Downloads { - fn github_api(&self) -> &GitHub { &self.github } + fn github_api(&self) -> &GitHub { + &self.github + } fn request(&self, r: &Request<'_>) -> Lookup<(Body, Headers)> { - self.calls.lock().unwrap().push((r.method.into(), r.url.into())); - if self.missing { return failed("HTTP 404: release does not exist"); } + self.calls + .lock() + .unwrap() + .push((r.method.into(), r.url.into())); + if self.missing { + return failed("HTTP 404: release does not exist"); + } if r.url.ends_with("CHECKSUMS") { assert_eq!(r.want, Want::Text); - let body = if self.bad_checksum { "123 5 wrong.tar.gz\n".into() } else { + let body = if self.bad_checksum { + "123 5 wrong.tar.gz\n".into() + } else { "123 5 homo_sapiens_vep_116_GRCh38.tar.gz\n456 6 homo_sapiens_vep_115_GRCh38.tar.gz\n".into() }; Ok((Body::Text(body), Headers::default())) } else { assert_eq!(r.method, "HEAD", "large archive body was requested"); - Ok((Body::None, Headers::from_pairs(vec![("Content-Length".into(), if self.bad_length { "1" } else { "7664082467" }.into())]))) + Ok(( + Body::None, + Headers::from_pairs(vec![( + "Content-Length".into(), + if self.bad_length { "1" } else { "7664082467" }.into(), + )]), + )) } } } fn check(missing: bool, bad_length: bool, bad_checksum: bool) { - let fetch = Downloads { github: GitHub { api: "https://invalid".into(), token: None }, missing, bad_length, bad_checksum, calls: Mutex::new(vec![]) }; - let registry = FakeRegistry { tags: |_| panic!("registry is not needed"), digests: |_| panic!("registry is not needed"), asked: Mutex::new(vec![]) }; - let pins = Pins::parse("VEP_CACHE_RELEASE=\"116\"\nPCGR_VEP_CACHE_RELEASE=\"115\"\nPCGR_DATA_BUNDLE=\"20260620\"\nPCGR_DATA_BYTES=\"7664082467\"\n"); + let fetch = Downloads { + github: GitHub { + api: "https://invalid".into(), + token: None, + }, + missing, + bad_length, + bad_checksum, + calls: Mutex::new(vec![]), + }; + let registry = FakeRegistry { + tags: |_| panic!("registry is not needed"), + digests: |_| panic!("registry is not needed"), + asked: Mutex::new(vec![]), + }; + let pins = Pins::parse( + "VEP_CACHE_RELEASE=\"116\"\nPCGR_VEP_CACHE_RELEASE=\"115\"\nPCGR_DATA_BUNDLE=\"20260620\"\nPCGR_DATA_BYTES=\"7664082467\"\n", + ); let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); let common = root.join("scripts/lib/common.sh"); - let c = Ctx { fetch: &fetch, registry: ®istry, pins: &pins, versions: Path::new("unused"), setup: Path::new("unused"), common: &common, now: "2026-11-15T00:00:00Z".parse().unwrap() }; - for name in ["VEP cache download", "CPSR cache download", "PCGR bundle download"] { - let (_, lookup) = DATABASES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("missing liveness row: {name}")); + let c = Ctx { + fetch: &fetch, + registry: ®istry, + pins: &pins, + versions: Path::new("unused"), + setup: Path::new("unused"), + common: &common, + now: "2026-11-15T00:00:00Z".parse().unwrap(), + }; + for name in [ + "VEP cache download", + "CPSR cache download", + "PCGR bundle download", + ] { + let (_, lookup) = DATABASES + .iter() + .find(|(n, _)| *n == name) + .unwrap_or_else(|| panic!("missing liveness row: {name}")); let got = lookup(&c); - let should_fail = missing || (bad_length && name == "PCGR bundle download") || (bad_checksum && name != "PCGR bundle download"); + let should_fail = missing + || (bad_length && name == "PCGR bundle download") + || (bad_checksum && name != "PCGR bundle download"); assert_eq!(got.is_err(), should_fail, "{name}: {got:?}"); } let calls = fetch.calls.lock().unwrap(); assert!(calls.iter().any(|(_, url)| url.contains("release-115/"))); } -#[test] fn clinical_links_head_only() { check(false, false, false); } -#[test] fn missing_release_fails() { check(true, false, false); } -#[test] fn changed_bundle_length_fails() { check(false, true, false); } -#[test] fn missing_cache_checksum_fails() { check(false, false, true); } +#[test] +fn clinical_links_head_only() { + check(false, false, false); +} +#[test] +fn missing_release_fails() { + check(true, false, false); +} +#[test] +fn changed_bundle_length_fails() { + check(false, true, false); +} +#[test] +fn missing_cache_checksum_fails() { + check(false, false, true); +} + +#[test] +#[ignore = "public network: archive HEADs and small Ensembl CHECKSUMS only"] +fn clinical_downloads_live() { + use og_fresh::http::Client; + use og_fresh::registry::Online; + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let versions = root.join("versions.env"); + let setup = root.join("scripts/setup.sh"); + let common = root.join("scripts/lib/common.sh"); + let client = Client::default(); + let registry = Online(&client); + let pins = Pins::read(&versions).unwrap(); + let c = Ctx { + fetch: &client, + registry: ®istry, + pins: &pins, + versions: &versions, + setup: &setup, + common: &common, + now: "2026-11-15T00:00:00Z".parse().unwrap(), + }; + for name in [ + "VEP cache download", + "CPSR cache download", + "PCGR bundle download", + ] { + let (_, lookup) = DATABASES.iter().find(|(n, _)| *n == name).unwrap(); + let found = lookup(&c).unwrap(); + println!("{}: {}", found.row.item, found.row.upstream); + } + let mut bad_pins = pins.clone(); + bad_pins + .values + .insert("VEP_CACHE_RELEASE".into(), "99999999".into()); + let bad = Ctx { + pins: &bad_pins, + ..c + }; + assert!( + og_fresh::lookups::vep_download(&bad).is_err(), + "nonexistent release accepted" + ); +} diff --git a/versions.env b/versions.env index 304024130..018f5c605 100644 --- a/versions.env +++ b/versions.env @@ -190,6 +190,8 @@ VEP_CACHE_RELEASE="116" # with PCGR_IMAGE. PCGR_VEP_CACHE_RELEASE="115" PCGR_DATA_BUNDLE="20260620" +# Exact archive size from the server; SHA-256 is measured from the full download. +PCGR_DATA_BYTES="7664082467" # pypgx-bundle git tag (step 32): must equal the version of PYPGX_IMAGE. PYPGX_BUNDLE_VERSION="0.26.0" # Cyrius (step 21, opt-in; legacy: no automated bumps) is installed by From cd1d2d82e0dc87977df8b06d996003a1fe054f40 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:18:19 +0100 Subject: [PATCH 50/86] fix(e2e): collect with og summary collect where two cases still ran the old CLI align-read-count-gate-1 and vendor-vcf-intake-2-nf-report ran bin/collect_summary.py as a command, which no longer has one: it exited 0 and wrote nothing. Also point stale comments at og summary collect and og traits check. --- CHANGELOG.md | 2 +- assets/genotype_rules.tsv | 5 +++-- bin/render_report.py | 4 ++-- scripts/17-cpsr.sh | 2 +- tests/e2e/align-read-count-gate-1.sh | 4 ++-- tests/e2e/vendor-vcf-intake-2-nf-report.sh | 12 ++++++------ 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cac8762..969285415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/assets/genotype_rules.tsv b/assets/genotype_rules.tsv index bd99e77bf..c4dd1b1e1 100644 --- a/assets/genotype_rules.tsv +++ b/assets/genotype_rules.tsv @@ -3,8 +3,9 @@ # Hand-curated from the primary papers in the pmids column; no text is taken from SNPedia or any # other database. GRCh38: ref is the GRCh38 base and alt the other allele the rule reads, both on # the plus strand as in dbSNP's GRCh38 record, also for a gene on the minus strand. -# scripts/ci/check-rule-alleles.sh checks every site against dbSNP; bin/traits_report.py -# --check-rules checks the rest (tests/test_traits_report.py runs both). +# scripts/ci/check-rule-alleles.sh checks every site against dbSNP; og traits check --table +# assets/traits.tsv --rules assets/genotype_rules.tsv checks the rest (tests/test_traits_report.py +# runs both). # Four kinds of line, tab separated, each with its own columns: # rule gene trait needs_phase ancestry note pmids # site rsid chrom pos ref alt (one per site, in the order the haplotypes use) diff --git a/bin/render_report.py b/bin/render_report.py index e949a759b..530cee487 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -127,7 +127,7 @@ def ancient_lines(d): def marker_state(m): - """A copy-aware marker's state in a few words (collect_summary.read_marker_table): + """A copy-aware marker's state in a few words (og summary collect, group_c/cn.rs read_marker_table): present or uncertain with the reads and the copies that carry it, not seen with the reads, too little depth, or not assessed.""" st = m.get("status") or "not assessed" @@ -2027,7 +2027,7 @@ def hemi(x): def two_hit_lines(t): - """The genes a called deletion and a ClinVar hit both reach (collect_summary.two_hits), and which deletion + """The genes a called deletion and a ClinVar hit both reach (og summary collect, two_hits.rs), and which deletion callers were read; none for a summary written before the rule.""" if not t: return [] diff --git a/scripts/17-cpsr.sh b/scripts/17-cpsr.sh index c2414a29b..fd3f0a9f9 100755 --- a/scripts/17-cpsr.sh +++ b/scripts/17-cpsr.sh @@ -65,7 +65,7 @@ CPSR_ID=$(sh "${PGP_ROOT}/bin/cpsr_sample_id" "$SAMPLE") # (CPSR_CLASSIFICATION), so 2.2's --classify_all is gone (2.3 refuses it). # The final class (CLASSIFICATION, source in ASSERTION_AUTHORITY) is # ClinVar's unless ClinVar has no record or a conflicted one: the default -# --clinvar_trust_level 0. The reports (bin/collect_summary.py) count the +# --clinvar_trust_level 0. The reports (og summary collect) count the # final class and list every row where CPSR's own class and ClinVar's # disagree on P/LP. diff --git a/tests/e2e/align-read-count-gate-1.sh b/tests/e2e/align-read-count-gate-1.sh index 0fe8e1dcb..3398a3cc0 100755 --- a/tests/e2e/align-read-count-gate-1.sh +++ b/tests/e2e/align-read-count-gate-1.sh @@ -50,7 +50,7 @@ TT="${T}/qc/${T}_read_count.tsv" check_eq "input_reads equals fastp's total_reads" "$(val input_reads "$TT")" "$FASTP" check_eq "the BAM's primary records equal fastp's total_reads" "$(primary "${T}/aligned/${T}_sorted.bam")" "$FASTP" check_eq "read_count_check" "$(val read_count_check "$TT")" ok -python3 "${REPO}/bin/collect_summary.py" --sample "$T" --sample-dir "${G}/${T}" --out "${CASE_TMP}/T.json" >/dev/null +"${REPO}/bin/og" summary collect --sample "$T" --sample-dir "${G}/${T}" --assets "${REPO}/assets" --out "${CASE_TMP}/T.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/T.json" -o "${CASE_TMP}/T.txt" >/dev/null check "the text report shows both counts" \ has "^ Alignment: ${FASTP} reads in, ${FASTP} primary records in the BAM\$" "$(cat "${CASE_TMP}/T.txt" 2>/dev/null)" @@ -141,7 +141,7 @@ check_eq "the published table: primary_records" "$(val primary_records "${R}/qc/ check_eq "the published table: missing_reads" "$(val missing_reads "${R}/qc/${X}_read_count.tsv")" 8000 check_eq "the published BAM's primary records" "$(primary "${R}/aligned/${X}_sorted.bam")" 24000 check "the table is not published beside the BAM" test ! -e "${G}/${R}/aligned/${X}_read_count.tsv" -python3 "${REPO}/bin/collect_summary.py" --sample "$X" --sample-dir "${G}/${R}" --out "${CASE_TMP}/X.json" >/dev/null +"${REPO}/bin/og" summary collect --sample "$X" --sample-dir "${G}/${R}" --assets "${REPO}/assets" --out "${CASE_TMP}/X.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.txt" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.html" >/dev/null check "the text report flags the missing reads" \ diff --git a/tests/e2e/vendor-vcf-intake-2-nf-report.sh b/tests/e2e/vendor-vcf-intake-2-nf-report.sh index 936730d4c..a3598ec60 100755 --- a/tests/e2e/vendor-vcf-intake-2-nf-report.sh +++ b/tests/e2e/vendor-vcf-intake-2-nf-report.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # A VCF-first Nextflow run (no BAM) with the tools a vendor-VCF user picks: # - the HTML report shows ROH, the mito haplogroup and CPIC, with the same -# ROH numbers and haplogroup bin/collect_summary.py (the bash report) reads +# ROH numbers and haplogroup og summary collect (the bash report) reads # from the same files, and it starts after the steps it shows; # - multiqc is selected but has nothing to read: the log says so in one line; # - the completion message prints, with no onComplete handler error; @@ -35,18 +35,18 @@ check "the report exists" test -s "$HTML" for h in 'Runs of Homozygosity' 'Mitochondrial Haplogroup' 'CPIC Drug Recommendations'; do check_eq "report card '${h}' is filled" "$(html_stat "$HTML" "$h" Status)" Complete done -python3 "${REPO}/bin/collect_summary.py" --sample "$SAMPLE" --sample-dir "$R" --out "${CASE_TMP}/summary.json" \ - > "${CASE_TMP}/summary.log" 2>&1 || cat "${CASE_TMP}/summary.log" +"${REPO}/bin/og" summary collect --sample "$SAMPLE" --sample-dir "$R" --assets "${REPO}/assets" \ + --out "${CASE_TMP}/summary.json" > "${CASE_TMP}/summary.log" 2>&1 || cat "${CASE_TMP}/summary.log" want() { python3 -c "import json, sys; d = json.load(open('${CASE_TMP}/summary.json'))['sections']$1; print($2)" 2>/dev/null; } ROH_TOTAL=$(want "['roh']['data']['total_mb']" "'%.1f MB' % d") ROH_LARGEST=$(want "['roh']['data']['largest_mb']" "'%.1f MB' % d") HG=$(want "['haplogroup']['data']['haplogroup']" "d") -echo "collect_summary.py: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" -check "collect_summary.py read the ROH file" test -n "$ROH_TOTAL" +echo "og summary collect: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" +check "og summary collect read the ROH file" test -n "$ROH_TOTAL" check_eq "ROH total matches the bash report's" "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH total')" "$ROH_TOTAL" check_eq "ROH largest segment matches the bash report's" \ "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH largest segment')" "$ROH_LARGEST" -check "collect_summary.py read the haplogroup" test -n "$HG" +check "og summary collect read the haplogroup" test -n "$HG" check_eq "haplogroup matches the bash report's" "$(html_stat "$HTML" 'Mitochondrial Haplogroup' Haplogroup)" "$HG" # The report waits for the steps it shows. From baced2682d4a0bb0cfd801aaa0e2762e2878c8f8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:19:39 +0100 Subject: [PATCH 51/86] build(rust): give every planned port its own crate, module and verb Adds the crates og-render, og-live, og-couple and og-admix, and every verb the 1.0.0 plan adds as an arm that calls its own module. Each such module is a stub that prints "og : not ported yet" and exits 2, so a port only replaces a module's body and never edits main.rs, the workspace member list or a crate's module list. A cargo test lists every planned verb and fails when one stops being routed. Bead: D1 Bead: D14 --- rust/Cargo.lock | 19 +++ rust/Cargo.toml | 6 +- rust/crates/og-admix/Cargo.toml | 12 ++ rust/crates/og-admix/src/lib.rs | 9 + rust/crates/og-admix/src/paint.rs | 2 + rust/crates/og-admix/src/project.rs | 2 + rust/crates/og-ancestry/src/compare.rs | 19 +++ rust/crates/og-ancestry/src/freqs.rs | 13 ++ rust/crates/og-ancestry/src/lib.rs | 17 ++ rust/crates/og-ancestry/src/paint.rs | 13 ++ rust/crates/og-ancestry/src/project.rs | 13 ++ rust/crates/og-ancestry/src/simulate.rs | 13 ++ rust/crates/og-couple/Cargo.toml | 12 ++ rust/crates/og-couple/src/genes.rs | 13 ++ rust/crates/og-couple/src/lib.rs | 8 + rust/crates/og-couple/src/screen.rs | 13 ++ rust/crates/og-dev/src/build/acmg_lof.rs | 13 ++ rust/crates/og-dev/src/build/mod.rs | 8 +- rust/crates/og-dev/src/build/roh_af.rs | 13 ++ rust/crates/og-dev/src/demo/ancestry_paint.rs | 2 + rust/crates/og-dev/src/demo/clinical.rs | 2 + .../og-dev/src/demo/clinical_clinvar.rs | 2 + rust/crates/og-dev/src/demo/family.rs | 1 + rust/crates/og-dev/src/demo/lineage.rs | 2 + rust/crates/og-dev/src/demo/mod.rs | 23 +++ rust/crates/og-dev/src/demo/pgx.rs | 2 + rust/crates/og-dev/src/demo/qc.rs | 2 + rust/crates/og-dev/src/demo/sv.rs | 2 + rust/crates/og-dev/src/demo/traits.rs | 1 + rust/crates/og-dev/src/demo_events.rs | 13 ++ rust/crates/og-dev/src/docs_checks.rs | 12 ++ rust/crates/og-dev/src/everything.rs | 13 ++ rust/crates/og-dev/src/lib.rs | 27 ++- rust/crates/og-dev/src/licences.rs | 13 ++ rust/crates/og-dev/src/params_doc.rs | 12 ++ rust/crates/og-dev/src/python_gate.rs | 13 ++ rust/crates/og-dev/src/steps_check.rs | 13 ++ rust/crates/og-dev/src/stub_http.rs | 13 ++ rust/crates/og-io/src/lib.rs | 3 + rust/crates/og-io/src/tabix_write.rs | 13 ++ rust/crates/og-live/Cargo.toml | 12 ++ rust/crates/og-live/src/auth.rs | 1 + rust/crates/og-live/src/charts.rs | 1 + rust/crates/og-live/src/csp.rs | 2 + rust/crates/og-live/src/demo.rs | 13 ++ rust/crates/og-live/src/family.rs | 12 ++ rust/crates/og-live/src/headlines.rs | 2 + rust/crates/og-live/src/ingest.rs | 2 + rust/crates/og-live/src/lib.rs | 30 ++++ rust/crates/og-live/src/map.rs | 2 + rust/crates/og-live/src/ops.rs | 2 + rust/crates/og-live/src/plugins.rs | 1 + rust/crates/og-live/src/recheck.rs | 2 + rust/crates/og-live/src/results/mod.rs | 12 ++ rust/crates/og-live/src/runs.rs | 2 + rust/crates/og-live/src/search.rs | 1 + rust/crates/og-live/src/server.rs | 13 ++ rust/crates/og-live/src/state.rs | 1 + rust/crates/og-live/src/static_files.rs | 2 + rust/crates/og-live/src/veil.rs | 2 + rust/crates/og-live/src/viewer.rs | 13 ++ rust/crates/og-live/src/widgets.rs | 2 + rust/crates/og-map/src/crossovers.rs | 2 + rust/crates/og-map/src/family.rs | 2 + rust/crates/og-map/src/layers/archaic.rs | 2 + rust/crates/og-map/src/layers/clinvar.rs | 2 + rust/crates/og-map/src/layers/mca.rs | 2 + rust/crates/og-map/src/layers/mod.rs | 5 + rust/crates/og-map/src/layers/paint.rs | 2 + rust/crates/og-map/src/layers/sv.rs | 2 + rust/crates/og-map/src/lib.rs | 5 + rust/crates/og-map/src/live.rs | 2 + rust/crates/og-map/src/svg.rs | 12 ++ rust/crates/og-map/src/threads.rs | 2 + rust/crates/og-pedigree/src/layout.rs | 2 + rust/crates/og-pedigree/src/lib.rs | 4 + rust/crates/og-render/Cargo.toml | 12 ++ rust/crates/og-render/src/charts/mod.rs | 13 ++ rust/crates/og-render/src/index.rs | 13 ++ rust/crates/og-render/src/lib.rs | 13 ++ rust/crates/og-render/src/render.rs | 13 ++ rust/crates/og-render/src/sections/acmg_sf.rs | 2 + .../crates/og-render/src/sections/ancestry.rs | 2 + .../og-render/src/sections/ancestry_fine.rs | 2 + rust/crates/og-render/src/sections/ancient.rs | 2 + rust/crates/og-render/src/sections/annotsv.rs | 2 + rust/crates/og-render/src/sections/apoe.rs | 1 + rust/crates/og-render/src/sections/archaic.rs | 2 + .../og-render/src/sections/bloodtype.rs | 2 + .../og-render/src/sections/callability.rs | 2 + rust/crates/og-render/src/sections/carrier.rs | 2 + rust/crates/og-render/src/sections/cdna.rs | 1 + .../og-render/src/sections/cdna_scan.rs | 2 + rust/crates/og-render/src/sections/chip.rs | 1 + rust/crates/og-render/src/sections/clinvar.rs | 2 + .../og-render/src/sections/cn_intervals.rs | 2 + rust/crates/og-render/src/sections/comphet.rs | 2 + rust/crates/og-render/src/sections/couple.rs | 2 + .../crates/og-render/src/sections/coverage.rs | 2 + rust/crates/og-render/src/sections/cpsr.rs | 1 + rust/crates/og-render/src/sections/cyp21a2.rs | 2 + .../og-render/src/sections/deep_intronic.rs | 2 + rust/crates/og-render/src/sections/dosage.rs | 2 + .../crates/og-render/src/sections/exomiser.rs | 2 + rust/crates/og-render/src/sections/g2.rs | 1 + .../og-render/src/sections/haplogroups.rs | 2 + rust/crates/og-render/src/sections/hba.rs | 1 + rust/crates/og-render/src/sections/hla.rs | 1 + .../og-render/src/sections/locityper.rs | 2 + rust/crates/og-render/src/sections/lpa.rs | 1 + rust/crates/og-render/src/sections/mei.rs | 1 + rust/crates/og-render/src/sections/mito.rs | 1 + rust/crates/og-render/src/sections/mocha.rs | 2 + rust/crates/og-render/src/sections/mod.rs | 54 ++++++ .../crates/og-render/src/sections/paralogs.rs | 2 + .../og-render/src/sections/parascopy.rs | 2 + rust/crates/og-render/src/sections/pgx.rs | 1 + rust/crates/og-render/src/sections/prs.rs | 1 + rust/crates/og-render/src/sections/qc.rs | 1 + rust/crates/og-render/src/sections/rfc1.rs | 1 + rust/crates/og-render/src/sections/roh.rs | 1 + rust/crates/og-render/src/sections/run.rs | 1 + rust/crates/og-render/src/sections/smn1.rs | 1 + rust/crates/og-render/src/sections/str.rs | 1 + .../og-render/src/sections/sv_callers.rs | 2 + .../crates/og-render/src/sections/telomere.rs | 2 + rust/crates/og-render/src/sections/traits.rs | 2 + .../crates/og-render/src/sections/two_hits.rs | 2 + .../crates/og-render/src/sections/variants.rs | 2 + rust/crates/og-render/src/sections/viral.rs | 2 + rust/crates/og-render/src/sections/vntyper.rs | 2 + rust/crates/og-render/src/sections/y_tree.rs | 2 + rust/crates/og-render/src/sections/ystr.rs | 1 + rust/crates/og-setup/src/lib.rs | 5 +- rust/crates/og-setup/src/revel.rs | 12 ++ rust/crates/og-setup/src/roh_af.rs | 12 ++ rust/crates/og-steps/src/lib.rs | 3 + rust/crates/og-steps/src/longread.rs | 13 ++ rust/crates/og-steps/src/sv.rs | 2 + .../crates/og-steps/src/sv/consensus_check.rs | 13 ++ rust/crates/og-steps/src/sv/survivor_prep.rs | 13 ++ rust/crates/og-summary/src/collect.rs | 1 + .../og-summary/src/collect/acmg_card.rs | 2 + rust/crates/og-vcf/src/lib.rs | 1 + rust/crates/og-vcf/src/shift.rs | 20 +++ rust/og/Cargo.toml | 3 + rust/og/src/main.rs | 90 ++++++++++ rust/og/tests/port_scaffold.rs | 158 ++++++++++++++++++ 148 files changed, 1123 insertions(+), 4 deletions(-) create mode 100644 rust/crates/og-admix/Cargo.toml create mode 100644 rust/crates/og-admix/src/lib.rs create mode 100644 rust/crates/og-admix/src/paint.rs create mode 100644 rust/crates/og-admix/src/project.rs create mode 100644 rust/crates/og-ancestry/src/compare.rs create mode 100644 rust/crates/og-ancestry/src/freqs.rs create mode 100644 rust/crates/og-ancestry/src/paint.rs create mode 100644 rust/crates/og-ancestry/src/project.rs create mode 100644 rust/crates/og-ancestry/src/simulate.rs create mode 100644 rust/crates/og-couple/Cargo.toml create mode 100644 rust/crates/og-couple/src/genes.rs create mode 100644 rust/crates/og-couple/src/lib.rs create mode 100644 rust/crates/og-couple/src/screen.rs create mode 100644 rust/crates/og-dev/src/build/acmg_lof.rs create mode 100644 rust/crates/og-dev/src/build/roh_af.rs create mode 100644 rust/crates/og-dev/src/demo/ancestry_paint.rs create mode 100644 rust/crates/og-dev/src/demo/clinical.rs create mode 100644 rust/crates/og-dev/src/demo/clinical_clinvar.rs create mode 100644 rust/crates/og-dev/src/demo/family.rs create mode 100644 rust/crates/og-dev/src/demo/lineage.rs create mode 100644 rust/crates/og-dev/src/demo/mod.rs create mode 100644 rust/crates/og-dev/src/demo/pgx.rs create mode 100644 rust/crates/og-dev/src/demo/qc.rs create mode 100644 rust/crates/og-dev/src/demo/sv.rs create mode 100644 rust/crates/og-dev/src/demo/traits.rs create mode 100644 rust/crates/og-dev/src/demo_events.rs create mode 100644 rust/crates/og-dev/src/docs_checks.rs create mode 100644 rust/crates/og-dev/src/everything.rs create mode 100644 rust/crates/og-dev/src/licences.rs create mode 100644 rust/crates/og-dev/src/params_doc.rs create mode 100644 rust/crates/og-dev/src/python_gate.rs create mode 100644 rust/crates/og-dev/src/steps_check.rs create mode 100644 rust/crates/og-dev/src/stub_http.rs create mode 100644 rust/crates/og-io/src/tabix_write.rs create mode 100644 rust/crates/og-live/Cargo.toml create mode 100644 rust/crates/og-live/src/auth.rs create mode 100644 rust/crates/og-live/src/charts.rs create mode 100644 rust/crates/og-live/src/csp.rs create mode 100644 rust/crates/og-live/src/demo.rs create mode 100644 rust/crates/og-live/src/family.rs create mode 100644 rust/crates/og-live/src/headlines.rs create mode 100644 rust/crates/og-live/src/ingest.rs create mode 100644 rust/crates/og-live/src/lib.rs create mode 100644 rust/crates/og-live/src/map.rs create mode 100644 rust/crates/og-live/src/ops.rs create mode 100644 rust/crates/og-live/src/plugins.rs create mode 100644 rust/crates/og-live/src/recheck.rs create mode 100644 rust/crates/og-live/src/results/mod.rs create mode 100644 rust/crates/og-live/src/runs.rs create mode 100644 rust/crates/og-live/src/search.rs create mode 100644 rust/crates/og-live/src/server.rs create mode 100644 rust/crates/og-live/src/state.rs create mode 100644 rust/crates/og-live/src/static_files.rs create mode 100644 rust/crates/og-live/src/veil.rs create mode 100644 rust/crates/og-live/src/viewer.rs create mode 100644 rust/crates/og-live/src/widgets.rs create mode 100644 rust/crates/og-map/src/crossovers.rs create mode 100644 rust/crates/og-map/src/family.rs create mode 100644 rust/crates/og-map/src/layers/archaic.rs create mode 100644 rust/crates/og-map/src/layers/clinvar.rs create mode 100644 rust/crates/og-map/src/layers/mca.rs create mode 100644 rust/crates/og-map/src/layers/paint.rs create mode 100644 rust/crates/og-map/src/layers/sv.rs create mode 100644 rust/crates/og-map/src/live.rs create mode 100644 rust/crates/og-map/src/svg.rs create mode 100644 rust/crates/og-map/src/threads.rs create mode 100644 rust/crates/og-pedigree/src/layout.rs create mode 100644 rust/crates/og-render/Cargo.toml create mode 100644 rust/crates/og-render/src/charts/mod.rs create mode 100644 rust/crates/og-render/src/index.rs create mode 100644 rust/crates/og-render/src/lib.rs create mode 100644 rust/crates/og-render/src/render.rs create mode 100644 rust/crates/og-render/src/sections/acmg_sf.rs create mode 100644 rust/crates/og-render/src/sections/ancestry.rs create mode 100644 rust/crates/og-render/src/sections/ancestry_fine.rs create mode 100644 rust/crates/og-render/src/sections/ancient.rs create mode 100644 rust/crates/og-render/src/sections/annotsv.rs create mode 100644 rust/crates/og-render/src/sections/apoe.rs create mode 100644 rust/crates/og-render/src/sections/archaic.rs create mode 100644 rust/crates/og-render/src/sections/bloodtype.rs create mode 100644 rust/crates/og-render/src/sections/callability.rs create mode 100644 rust/crates/og-render/src/sections/carrier.rs create mode 100644 rust/crates/og-render/src/sections/cdna.rs create mode 100644 rust/crates/og-render/src/sections/cdna_scan.rs create mode 100644 rust/crates/og-render/src/sections/chip.rs create mode 100644 rust/crates/og-render/src/sections/clinvar.rs create mode 100644 rust/crates/og-render/src/sections/cn_intervals.rs create mode 100644 rust/crates/og-render/src/sections/comphet.rs create mode 100644 rust/crates/og-render/src/sections/couple.rs create mode 100644 rust/crates/og-render/src/sections/coverage.rs create mode 100644 rust/crates/og-render/src/sections/cpsr.rs create mode 100644 rust/crates/og-render/src/sections/cyp21a2.rs create mode 100644 rust/crates/og-render/src/sections/deep_intronic.rs create mode 100644 rust/crates/og-render/src/sections/dosage.rs create mode 100644 rust/crates/og-render/src/sections/exomiser.rs create mode 100644 rust/crates/og-render/src/sections/g2.rs create mode 100644 rust/crates/og-render/src/sections/haplogroups.rs create mode 100644 rust/crates/og-render/src/sections/hba.rs create mode 100644 rust/crates/og-render/src/sections/hla.rs create mode 100644 rust/crates/og-render/src/sections/locityper.rs create mode 100644 rust/crates/og-render/src/sections/lpa.rs create mode 100644 rust/crates/og-render/src/sections/mei.rs create mode 100644 rust/crates/og-render/src/sections/mito.rs create mode 100644 rust/crates/og-render/src/sections/mocha.rs create mode 100644 rust/crates/og-render/src/sections/mod.rs create mode 100644 rust/crates/og-render/src/sections/paralogs.rs create mode 100644 rust/crates/og-render/src/sections/parascopy.rs create mode 100644 rust/crates/og-render/src/sections/pgx.rs create mode 100644 rust/crates/og-render/src/sections/prs.rs create mode 100644 rust/crates/og-render/src/sections/qc.rs create mode 100644 rust/crates/og-render/src/sections/rfc1.rs create mode 100644 rust/crates/og-render/src/sections/roh.rs create mode 100644 rust/crates/og-render/src/sections/run.rs create mode 100644 rust/crates/og-render/src/sections/smn1.rs create mode 100644 rust/crates/og-render/src/sections/str.rs create mode 100644 rust/crates/og-render/src/sections/sv_callers.rs create mode 100644 rust/crates/og-render/src/sections/telomere.rs create mode 100644 rust/crates/og-render/src/sections/traits.rs create mode 100644 rust/crates/og-render/src/sections/two_hits.rs create mode 100644 rust/crates/og-render/src/sections/variants.rs create mode 100644 rust/crates/og-render/src/sections/viral.rs create mode 100644 rust/crates/og-render/src/sections/vntyper.rs create mode 100644 rust/crates/og-render/src/sections/y_tree.rs create mode 100644 rust/crates/og-render/src/sections/ystr.rs create mode 100644 rust/crates/og-setup/src/revel.rs create mode 100644 rust/crates/og-setup/src/roh_af.rs create mode 100644 rust/crates/og-steps/src/longread.rs create mode 100644 rust/crates/og-steps/src/sv/consensus_check.rs create mode 100644 rust/crates/og-steps/src/sv/survivor_prep.rs create mode 100644 rust/crates/og-summary/src/collect/acmg_card.rs create mode 100644 rust/crates/og-vcf/src/shift.rs create mode 100644 rust/og/tests/port_scaffold.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0dbef91fe..6bf5d012d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -603,16 +603,19 @@ dependencies = [ "og-chip", "og-clinvar", "og-cn", + "og-couple", "og-dev", "og-filters", "og-fresh", "og-intervals", "og-io", + "og-live", "og-lookup", "og-map", "og-pgx", "og-qc", "og-recheck", + "og-render", "og-reports", "og-runlog", "og-setup", @@ -622,6 +625,10 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-admix" +version = "0.1.0" + [[package]] name = "og-ancestry" version = "0.1.0" @@ -687,6 +694,10 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-couple" +version = "0.1.0" + [[package]] name = "og-dev" version = "0.1.0" @@ -738,6 +749,10 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "og-live" +version = "0.1.0" + [[package]] name = "og-lookup" version = "0.1.0" @@ -806,6 +821,10 @@ dependencies = [ "shlex 1.3.0", ] +[[package]] +name = "og-render" +version = "0.1.0" + [[package]] name = "og-reports" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2fa060ac3..416ee70c9 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "crates/og-render", "crates/og-live", "crates/og-couple", "crates/og-admix", "og"] [workspace.package] version = "0.1.0" @@ -35,6 +35,10 @@ og-summary = { path = "crates/og-summary" } og-lookup = { path = "crates/og-lookup" } og-fresh = { path = "crates/og-fresh" } og-recheck = { path = "crates/og-recheck" } +og-render = { path = "crates/og-render" } +og-live = { path = "crates/og-live" } +og-couple = { path = "crates/og-couple" } +og-admix = { path = "crates/og-admix" } indexmap = "2" flate2 = "1" memchr = "2" diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml new file mode 100644 index 000000000..784875681 --- /dev/null +++ b/rust/crates/og-admix/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-admix" +description = "What the fine-ancestry engines share: our own local-ancestry painting and the projection of ancestry fractions" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-admix/src/lib.rs b/rust/crates/og-admix/src/lib.rs new file mode 100644 index 000000000..bf0a44088 --- /dev/null +++ b/rust/crates/og-admix/src/lib.rs @@ -0,0 +1,9 @@ +//! What the fine-ancestry engines of step 26b share: our own +//! local-ancestry painting ([`paint`]) and the projection of ancestry +//! fractions on the panel's frequencies ([`project`]). +//! +//! Empty until their units of the 1.0.0 plan land. The crate exists now so +//! those units add code without editing the same files. + +pub mod paint; +pub mod project; diff --git a/rust/crates/og-admix/src/paint.rs b/rust/crates/og-admix/src/paint.rs new file mode 100644 index 000000000..1fc0f571c --- /dev/null +++ b/rust/crates/og-admix/src/paint.rs @@ -0,0 +1,2 @@ +//! Local-ancestry painting, our own engine in place of RFMix. Empty until +//! its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-admix/src/project.rs b/rust/crates/og-admix/src/project.rs new file mode 100644 index 000000000..39710102e --- /dev/null +++ b/rust/crates/og-admix/src/project.rs @@ -0,0 +1,2 @@ +//! Ancestry fractions projected on the panel's frequencies, in place of +//! ADMIXTURE. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-ancestry/src/compare.rs b/rust/crates/og-ancestry/src/compare.rs new file mode 100644 index 000000000..426305ebe --- /dev/null +++ b/rust/crates/og-ancestry/src/compare.rs @@ -0,0 +1,19 @@ +//! `og archaic compare` and `og archaic genomes`: step 43's archaic totals +//! beside published calls, which `bin/archaic_compare.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, each verb says +//! so on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og archaic compare ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn compare_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og archaic compare: not ported yet"); + 2 +} + +/// `og archaic genomes ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn genomes_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og archaic genomes: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/freqs.rs b/rust/crates/og-ancestry/src/freqs.rs new file mode 100644 index 000000000..5a88a4dc8 --- /dev/null +++ b/rust/crates/og-ancestry/src/freqs.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine freqs`: each reference group's allele frequencies, +//! counted from the panel. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine freqs ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine freqs: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 74febd042..5ae862961 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -10,6 +10,10 @@ //! - [`chroms`]: the chromosomes long enough for RFMix to paint. //! - [`report`]: the fractions table, the painting (BED and PNG) and the //! plain-text summary. +//! - [`freqs`], [`project`], [`paint`], [`simulate`]: `og ancestry-fine +//! freqs|project|paint|simulate`, the engines that replace ADMIXTURE and +//! RFMix, and [`compare`]: `og archaic compare|genomes`. Not ported yet: +//! each says so and exits 2. //! //! Each command writes what `bin/ancestry_fine_report.py` wrote (checked by //! `tests/parity`), with three deliberate changes: every output is written @@ -25,10 +29,15 @@ pub mod ancient; pub mod archaic; pub mod chroms; +pub mod compare; +pub mod freqs; pub mod order; +pub mod paint; pub mod panel; +pub mod project; pub mod report; pub mod sample_bed; +pub mod simulate; use std::fmt; use std::io; @@ -557,6 +566,14 @@ pub fn cli(args: &[String], out: &mut dyn io::Write, err: &mut dyn io::Write) -> return 2; }; let rest = &args[1..]; + // The engines read their own command lines, outside the Spec table. + match verb.as_str() { + "freqs" => return freqs::cli(rest, out, err), + "project" => return project::cli(rest, out, err), + "paint" => return paint::cli(rest, out, err), + "simulate" => return simulate::cli(rest, out, err), + _ => {} + } let (spec, run): (&Spec, Run) = match verb.as_str() { "panel-map" => (&panel::PANEL_MAP, panel::panel_map_main), "check-map" => (&panel::CHECK_MAP, panel::check_map_main), diff --git a/rust/crates/og-ancestry/src/paint.rs b/rust/crates/og-ancestry/src/paint.rs new file mode 100644 index 000000000..10e8df969 --- /dev/null +++ b/rust/crates/og-ancestry/src/paint.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine paint`: the local-ancestry painting, in place of +//! RFMix. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine paint ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine paint: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/project.rs b/rust/crates/og-ancestry/src/project.rs new file mode 100644 index 000000000..ff633027b --- /dev/null +++ b/rust/crates/og-ancestry/src/project.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine project`: a person's ancestry fractions projected on +//! the panel's frequencies, in place of ADMIXTURE. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine project ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine project: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/simulate.rs b/rust/crates/og-ancestry/src/simulate.rs new file mode 100644 index 000000000..bccf46ec0 --- /dev/null +++ b/rust/crates/og-ancestry/src/simulate.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine simulate`: simulated admixed genomes that check the +//! painting and the projection. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine simulate ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine simulate: not ported yet"); + 2 +} diff --git a/rust/crates/og-couple/Cargo.toml b/rust/crates/og-couple/Cargo.toml new file mode 100644 index 000000000..7ece1392f --- /dev/null +++ b/rust/crates/og-couple/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-couple" +description = "og couple screen|genes: step 39's couple carrier screen" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-couple/src/genes.rs b/rust/crates/og-couple/src/genes.rs new file mode 100644 index 000000000..22511fb6a --- /dev/null +++ b/rust/crates/og-couple/src/genes.rs @@ -0,0 +1,13 @@ +//! `og couple genes`: the gene list of step 39's screen, which +//! `bin/couple_report.py genes` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og couple genes ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og couple genes: not ported yet"); + 2 +} diff --git a/rust/crates/og-couple/src/lib.rs b/rust/crates/og-couple/src/lib.rs new file mode 100644 index 000000000..0ae5dbe68 --- /dev/null +++ b/rust/crates/og-couple/src/lib.rs @@ -0,0 +1,8 @@ +//! Step 39, the couple carrier screen: `og couple screen` ([`screen`]) and +//! `og couple genes` ([`genes`]). +//! +//! Not ported yet: each verb says so and exits 2 until its unit of the +//! 1.0.0 plan lands. + +pub mod genes; +pub mod screen; diff --git a/rust/crates/og-couple/src/screen.rs b/rust/crates/og-couple/src/screen.rs new file mode 100644 index 000000000..d86757436 --- /dev/null +++ b/rust/crates/og-couple/src/screen.rs @@ -0,0 +1,13 @@ +//! `og couple screen`: step 39's couple carrier screen, which +//! `bin/couple_report.py screen` runs today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og couple screen ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og couple screen: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/build/acmg_lof.rs b/rust/crates/og-dev/src/build/acmg_lof.rs new file mode 100644 index 000000000..7cab0cd81 --- /dev/null +++ b/rust/crates/og-dev/src/build/acmg_lof.rs @@ -0,0 +1,13 @@ +//! `og dev build acmg-lof`: the loss-of-function column of +//! `assets/acmg_sf.tsv`, built from pinned ClinGen scores. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev build acmg-lof ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev build acmg-lof: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/build/mod.rs b/rust/crates/og-dev/src/build/mod.rs index d8e9c5931..f0471e546 100644 --- a/rust/crates/og-dev/src/build/mod.rs +++ b/rust/crates/og-dev/src/build/mod.rs @@ -11,6 +11,8 @@ //! `assets/callability_genes.tsv` (`scripts/ci/build-callability.sh`). //! - `og dev build sv-rule-regions` ([`sv_rules`]): //! `assets/sv_rule_regions.tsv` (`scripts/ci/build-sv-rule-regions.sh`). +//! - `og dev build roh-af` ([`roh_af`]) and `og dev build acmg-lof` +//! ([`acmg_lof`]): not ported yet, each says so and exits 2. //! //! Exit 0 when the file was written (or `--check` found it right); 1 when //! `--check` found it different, or when the callability or SV rule @@ -19,9 +21,11 @@ //! exited; 2 for a usage error or an input that cannot be read, and for //! every cn-intervals input the rules refuse, as its Python builder exited. +pub mod acmg_lof; pub mod callability; pub mod cn_intervals; mod gtf; +pub mod roh_af; pub mod sv_rules; use og_io::{Lines, Utf8}; @@ -29,7 +33,7 @@ use std::fmt; use std::io::{self, BufRead, Write}; use std::path::Path; -const USAGE: &str = "usage: og dev build cn-intervals --clingen BED --segdups BED --gaps BED --satellites BED --parascopy TXT (--catalog FILE [--check] | --stdout)\n og dev build callability --work DIR --repo DIR --out FILE\n og dev build sv-rule-regions --work DIR --out FILE\n"; +const USAGE: &str = "usage: og dev build cn-intervals --clingen BED --segdups BED --gaps BED --satellites BED --parascopy TXT (--catalog FILE [--check] | --stdout)\n og dev build callability --work DIR --repo DIR --out FILE\n og dev build sv-rule-regions --work DIR --out FILE\n og dev build roh-af [args]\n og dev build acmg-lof [args]\n"; /// GRCh38's primary chromosomes in their sort order: chr1-22, X, Y. pub(crate) const CHROMS: [&str; 24] = [ @@ -194,6 +198,8 @@ pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { Some("cn-intervals") => cn_intervals::cli(&args[1..], out, err), Some("callability") => callability::cli(&args[1..], err), Some("sv-rule-regions") => sv_rules::cli(&args[1..], err), + Some("roh-af") => Ok(roh_af::cli(&args[1..], out, err)), + Some("acmg-lof") => Ok(acmg_lof::cli(&args[1..], out, err)), Some("-h" | "--help") => { let _ = out.write_all(USAGE.as_bytes()); return 0; diff --git a/rust/crates/og-dev/src/build/roh_af.rs b/rust/crates/og-dev/src/build/roh_af.rs new file mode 100644 index 000000000..a07b95d42 --- /dev/null +++ b/rust/crates/og-dev/src/build/roh_af.rs @@ -0,0 +1,13 @@ +//! `og dev build roh-af`: the ROH allele-frequency panel, built from a +//! pinned gnomAD release. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev build roh-af ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev build roh-af: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/demo/ancestry_paint.rs b/rust/crates/og-dev/src/demo/ancestry_paint.rs new file mode 100644 index 000000000..49494c61e --- /dev/null +++ b/rust/crates/og-dev/src/demo/ancestry_paint.rs @@ -0,0 +1,2 @@ +//! The demo sample's ancestry painting. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/clinical.rs b/rust/crates/og-dev/src/demo/clinical.rs new file mode 100644 index 000000000..62a29a193 --- /dev/null +++ b/rust/crates/og-dev/src/demo/clinical.rs @@ -0,0 +1,2 @@ +//! The demo sample's secondary findings. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/clinical_clinvar.rs b/rust/crates/og-dev/src/demo/clinical_clinvar.rs new file mode 100644 index 000000000..d7cbe0aff --- /dev/null +++ b/rust/crates/og-dev/src/demo/clinical_clinvar.rs @@ -0,0 +1,2 @@ +//! The demo sample's ClinVar results. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/family.rs b/rust/crates/og-dev/src/demo/family.rs new file mode 100644 index 000000000..5732c8364 --- /dev/null +++ b/rust/crates/og-dev/src/demo/family.rs @@ -0,0 +1 @@ +//! The demo family. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo/lineage.rs b/rust/crates/og-dev/src/demo/lineage.rs new file mode 100644 index 000000000..b2b79df2a --- /dev/null +++ b/rust/crates/og-dev/src/demo/lineage.rs @@ -0,0 +1,2 @@ +//! The demo sample's ancestry and lineage. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo/mod.rs b/rust/crates/og-dev/src/demo/mod.rs new file mode 100644 index 000000000..1ff961bca --- /dev/null +++ b/rust/crates/og-dev/src/demo/mod.rs @@ -0,0 +1,23 @@ +//! `og dev demo-sample`: the made-up demo sample the docs, the pictures and +//! the live demo use, which `tests/demo/make_demo_sample.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +pub mod ancestry_paint; +pub mod clinical; +pub mod clinical_clinvar; +pub mod family; +pub mod lineage; +pub mod pgx; +pub mod qc; +pub mod sv; +pub mod traits; + +use std::io::Write; + +/// `og dev demo-sample ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev demo-sample: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/demo/pgx.rs b/rust/crates/og-dev/src/demo/pgx.rs new file mode 100644 index 000000000..f8cad997b --- /dev/null +++ b/rust/crates/og-dev/src/demo/pgx.rs @@ -0,0 +1,2 @@ +//! The demo sample's pharmacogenomics. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/qc.rs b/rust/crates/og-dev/src/demo/qc.rs new file mode 100644 index 000000000..5520120bf --- /dev/null +++ b/rust/crates/og-dev/src/demo/qc.rs @@ -0,0 +1,2 @@ +//! The demo sample's quality tables. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-dev/src/demo/sv.rs b/rust/crates/og-dev/src/demo/sv.rs new file mode 100644 index 000000000..41d8a3e6c --- /dev/null +++ b/rust/crates/og-dev/src/demo/sv.rs @@ -0,0 +1,2 @@ +//! The demo sample's structural variants and repeats. Empty until its unit +//! of the 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo/traits.rs b/rust/crates/og-dev/src/demo/traits.rs new file mode 100644 index 000000000..fae44a955 --- /dev/null +++ b/rust/crates/og-dev/src/demo/traits.rs @@ -0,0 +1 @@ +//! The demo sample's traits. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo_events.rs b/rust/crates/og-dev/src/demo_events.rs new file mode 100644 index 000000000..7ff3320b0 --- /dev/null +++ b/rust/crates/og-dev/src/demo_events.rs @@ -0,0 +1,13 @@ +//! `og dev demo-events`: the replayed event log of the live demo, which +//! `tests/demo/make_demo_events.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev demo-events ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev demo-events: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/docs_checks.rs b/rust/crates/og-dev/src/docs_checks.rs new file mode 100644 index 000000000..e4593ad08 --- /dev/null +++ b/rust/crates/og-dev/src/docs_checks.rs @@ -0,0 +1,12 @@ +//! `og dev docs-check`: the checks that keep the docs true to the code. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev docs-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev docs-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/everything.rs b/rust/crates/og-dev/src/everything.rs new file mode 100644 index 000000000..be8e755e5 --- /dev/null +++ b/rust/crates/og-dev/src/everything.rs @@ -0,0 +1,13 @@ +//! `og dev everything-check`: `assets/everything.tsv` against every tool, +//! switch and installer. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev everything-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev everything-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index a5f776c46..d2f16ba86 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -33,32 +33,48 @@ //! found, against every pin in the repository. //! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): //! rebuild a committed asset table from its pinned downloads. +//! - Not ported yet, each says so and exits 2: `og dev demo-sample` +//! ([`demo`]), `og dev demo-events` ([`demo_events`]), `og dev params-doc` +//! ([`params_doc`]), `og dev python-gate` ([`python_gate`]), +//! `og dev everything-check` ([`everything`]), `og dev licences` +//! ([`licences`]), `og dev docs-check` ([`docs_checks`]), +//! `og dev stub-http` ([`stub_http`]) and `og dev steps-check` +//! ([`steps_check`]). pub mod build; pub mod contract; +pub mod demo; +pub mod demo_events; +pub mod docs_checks; +pub mod everything; pub mod evidence; pub mod fixture; pub mod helpers; pub mod images; +pub mod licences; mod lookaround; pub mod nf_trace; +pub mod params_doc; pub mod parity; pub mod personal_data; pub mod pgs; pub mod pharmcat; +pub mod python_gate; pub mod remnants; pub mod renovate; pub mod report_only; pub mod schema; pub mod shlex; pub mod smoke; +pub mod steps_check; +pub mod stub_http; pub mod versions; pub mod versions_env; mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions|roh-af|acmg-lof [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]\n og dev demo-sample [args]\n og dev demo-events [args]\n og dev params-doc [args]\n og dev python-gate [args]\n og dev everything-check [args]\n og dev licences [args]\n og dev docs-check [args]\n og dev stub-http [args]\n og dev steps-check [args]"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -182,6 +198,15 @@ pub fn main(args: &[String]) -> u8 { Some("smoke") => run(smoke::cli, &args[1..]), Some("build") => run(build::cli, &args[1..]), Some("helpers") => run(helpers::cli, &args[1..]), + Some("demo-sample") => run(demo::cli, &args[1..]), + Some("demo-events") => run(demo_events::cli, &args[1..]), + Some("params-doc") => run(params_doc::cli, &args[1..]), + Some("python-gate") => run(python_gate::cli, &args[1..]), + Some("everything-check") => run(everything::cli, &args[1..]), + Some("licences") => run(licences::cli, &args[1..]), + Some("docs-check") => run(docs_checks::cli, &args[1..]), + Some("stub-http") => run(stub_http::cli, &args[1..]), + Some("steps-check") => run(steps_check::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/rust/crates/og-dev/src/licences.rs b/rust/crates/og-dev/src/licences.rs new file mode 100644 index 000000000..58319e4e5 --- /dev/null +++ b/rust/crates/og-dev/src/licences.rs @@ -0,0 +1,13 @@ +//! `og dev licences`: the licences of the data and code OwnGenome ships, +//! and the notices of the libraries in the binary. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev licences ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev licences: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/params_doc.rs b/rust/crates/og-dev/src/params_doc.rs new file mode 100644 index 000000000..c4cd72b4e --- /dev/null +++ b/rust/crates/og-dev/src/params_doc.rs @@ -0,0 +1,12 @@ +//! `og dev params-doc`: every schema parameter against its row in the docs. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev params-doc ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev params-doc: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/python_gate.rs b/rust/crates/og-dev/src/python_gate.rs new file mode 100644 index 000000000..bbd69d72f --- /dev/null +++ b/rust/crates/og-dev/src/python_gate.rs @@ -0,0 +1,13 @@ +//! `og dev python-gate`: every Python file, heredoc and `python3 -c` call +//! against the allow list. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev python-gate ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev python-gate: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/steps_check.rs b/rust/crates/og-dev/src/steps_check.rs new file mode 100644 index 000000000..0d162ab6d --- /dev/null +++ b/rust/crates/og-dev/src/steps_check.rs @@ -0,0 +1,13 @@ +//! `og dev steps-check`: `assets/steps.tsv` against `run-all.sh`, `main.nf` +//! and the modules. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev steps-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev steps-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/stub_http.rs b/rust/crates/og-dev/src/stub_http.rs new file mode 100644 index 000000000..9d7ce06ee --- /dev/null +++ b/rust/crates/og-dev/src/stub_http.rs @@ -0,0 +1,13 @@ +//! `og dev stub-http`: a local HTTP server with scripted failures, for the +//! shell tests of download retries. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev stub-http ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev stub-http: not ported yet"); + 2 +} diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs index 1943c18be..d5d3d46fa 100644 --- a/rust/crates/og-io/src/lib.rs +++ b/rust/crates/og-io/src/lib.rs @@ -11,9 +11,12 @@ //! BGZF writer that compresses on many cores. //! - [`tabix`]: the lines of a region of a bgzipped file, through its //! `.tbi` or `.csi` index, as `tabix FILE seq:beg-end` prints them. +//! - [`tabix_write`]: `og vcf index`, a `.tbi` for a BGZF VCF (not ported +//! yet). pub mod bgzf; pub mod tabix; +pub mod tabix_write; use flate2::bufread::GzDecoder; use std::fs::{self, File}; diff --git a/rust/crates/og-io/src/tabix_write.rs b/rust/crates/og-io/src/tabix_write.rs new file mode 100644 index 000000000..e395a1d78 --- /dev/null +++ b/rust/crates/og-io/src/tabix_write.rs @@ -0,0 +1,13 @@ +//! `og vcf index`: a `.tbi` index for a BGZF VCF, which step 38 writes +//! today with `pysam.tabix_index`. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og vcf index ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og vcf index: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml new file mode 100644 index 000000000..1e1e7ad65 --- /dev/null +++ b/rust/crates/og-live/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-live" +description = "og live serve|results|family|demo and og view serve: the live site, its pages and the reads viewer" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-live/src/auth.rs b/rust/crates/og-live/src/auth.rs new file mode 100644 index 000000000..9123ad3d5 --- /dev/null +++ b/rust/crates/og-live/src/auth.rs @@ -0,0 +1 @@ +//! Sign-in for the live site. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/charts.rs b/rust/crates/og-live/src/charts.rs new file mode 100644 index 000000000..63070ae95 --- /dev/null +++ b/rust/crates/og-live/src/charts.rs @@ -0,0 +1 @@ +//! The live site's charts. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/csp.rs b/rust/crates/og-live/src/csp.rs new file mode 100644 index 000000000..a3a75eb2d --- /dev/null +++ b/rust/crates/og-live/src/csp.rs @@ -0,0 +1,2 @@ +//! The live site's Content-Security-Policy. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/demo.rs b/rust/crates/og-live/src/demo.rs new file mode 100644 index 000000000..94096e9cb --- /dev/null +++ b/rust/crates/og-live/src/demo.rs @@ -0,0 +1,13 @@ +//! `og live demo`: the public demo as static pages, which +//! `tests/demo/build_live_demo.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live demo ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live demo: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/family.rs b/rust/crates/og-live/src/family.rs new file mode 100644 index 000000000..ab31e08b1 --- /dev/null +++ b/rust/crates/og-live/src/family.rs @@ -0,0 +1,12 @@ +//! `og live family`: the live site's family page. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live family ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live family: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/headlines.rs b/rust/crates/og-live/src/headlines.rs new file mode 100644 index 000000000..8757bbc4d --- /dev/null +++ b/rust/crates/og-live/src/headlines.rs @@ -0,0 +1,2 @@ +//! The headlines of a results page. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/ingest.rs b/rust/crates/og-live/src/ingest.rs new file mode 100644 index 000000000..24f2b8b32 --- /dev/null +++ b/rust/crates/og-live/src/ingest.rs @@ -0,0 +1,2 @@ +//! Runs taken into the live site. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/lib.rs b/rust/crates/og-live/src/lib.rs new file mode 100644 index 000000000..acd150a9c --- /dev/null +++ b/rust/crates/og-live/src/lib.rs @@ -0,0 +1,30 @@ +//! The live site: its server (`og live serve`, [`server`]), each person's +//! results page (`og live results`, [`results`]), the family page +//! (`og live family`, [`family`]), the public demo as static pages +//! (`og live demo`, [`demo`]) and the reads viewer (`og view serve`, +//! [`viewer`]). +//! +//! Not ported yet: each verb says so and exits 2 until its unit of the +//! 1.0.0 plan lands. Every module those units name is declared here now, +//! so they add code without editing the same files. + +pub mod auth; +pub mod charts; +pub mod csp; +pub mod demo; +pub mod family; +pub mod headlines; +pub mod ingest; +pub mod map; +pub mod ops; +pub mod plugins; +pub mod recheck; +pub mod results; +pub mod runs; +pub mod search; +pub mod server; +pub mod state; +pub mod static_files; +pub mod veil; +pub mod viewer; +pub mod widgets; diff --git a/rust/crates/og-live/src/map.rs b/rust/crates/og-live/src/map.rs new file mode 100644 index 000000000..288f8a36f --- /dev/null +++ b/rust/crates/og-live/src/map.rs @@ -0,0 +1,2 @@ +//! The genome map on the live site. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/ops.rs b/rust/crates/og-live/src/ops.rs new file mode 100644 index 000000000..0b4d3ea7b --- /dev/null +++ b/rust/crates/og-live/src/ops.rs @@ -0,0 +1,2 @@ +//! The live site's run progress and estimates. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/plugins.rs b/rust/crates/og-live/src/plugins.rs new file mode 100644 index 000000000..e6939bce0 --- /dev/null +++ b/rust/crates/og-live/src/plugins.rs @@ -0,0 +1 @@ +//! The live site's plugins. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/recheck.rs b/rust/crates/og-live/src/recheck.rs new file mode 100644 index 000000000..9c9f1bd55 --- /dev/null +++ b/rust/crates/og-live/src/recheck.rs @@ -0,0 +1,2 @@ +//! The recheck view of the live site. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-live/src/results/mod.rs b/rust/crates/og-live/src/results/mod.rs new file mode 100644 index 000000000..072f085f0 --- /dev/null +++ b/rust/crates/og-live/src/results/mod.rs @@ -0,0 +1,12 @@ +//! `og live results`: each person's results page, which +//! `bin/live_results.py` writes today. +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live results ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live results: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/runs.rs b/rust/crates/og-live/src/runs.rs new file mode 100644 index 000000000..2b7502103 --- /dev/null +++ b/rust/crates/og-live/src/runs.rs @@ -0,0 +1,2 @@ +//! The runs the live site knows. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/search.rs b/rust/crates/og-live/src/search.rs new file mode 100644 index 000000000..a4e302f07 --- /dev/null +++ b/rust/crates/og-live/src/search.rs @@ -0,0 +1 @@ +//! The live site's search. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/server.rs b/rust/crates/og-live/src/server.rs new file mode 100644 index 000000000..b15cb7cb8 --- /dev/null +++ b/rust/crates/og-live/src/server.rs @@ -0,0 +1,13 @@ +//! `og live serve`: the live site's server, which `bin/owngenome_live.py` +//! runs today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live serve ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live serve: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/state.rs b/rust/crates/og-live/src/state.rs new file mode 100644 index 000000000..abe3b071f --- /dev/null +++ b/rust/crates/og-live/src/state.rs @@ -0,0 +1 @@ +//! The live site's run state. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/static_files.rs b/rust/crates/og-live/src/static_files.rs new file mode 100644 index 000000000..71d0806aa --- /dev/null +++ b/rust/crates/og-live/src/static_files.rs @@ -0,0 +1,2 @@ +//! The live site's static files. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/veil.rs b/rust/crates/og-live/src/veil.rs new file mode 100644 index 000000000..be15c34c5 --- /dev/null +++ b/rust/crates/og-live/src/veil.rs @@ -0,0 +1,2 @@ +//! What a results page keeps closed until it is opened. Empty until its +//! unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/viewer.rs b/rust/crates/og-live/src/viewer.rs new file mode 100644 index 000000000..c3b9d3847 --- /dev/null +++ b/rust/crates/og-live/src/viewer.rs @@ -0,0 +1,13 @@ +//! `og view serve`: igv.js and one sample's reads and calls on 127.0.0.1, +//! which `assets/viewer/serve.py` serves today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og view serve ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og view serve: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/widgets.rs b/rust/crates/og-live/src/widgets.rs new file mode 100644 index 000000000..c3d21640f --- /dev/null +++ b/rust/crates/og-live/src/widgets.rs @@ -0,0 +1,2 @@ +//! The widgets of a results page. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-map/src/crossovers.rs b/rust/crates/og-map/src/crossovers.rs new file mode 100644 index 000000000..5f18284e1 --- /dev/null +++ b/rust/crates/og-map/src/crossovers.rs @@ -0,0 +1,2 @@ +//! The crossovers a child's genome map draws. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/family.rs b/rust/crates/og-map/src/family.rs new file mode 100644 index 000000000..7f4cf1052 --- /dev/null +++ b/rust/crates/og-map/src/family.rs @@ -0,0 +1,2 @@ +//! The family view of the genome map. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-map/src/layers/archaic.rs b/rust/crates/og-map/src/layers/archaic.rs new file mode 100644 index 000000000..960675be8 --- /dev/null +++ b/rust/crates/og-map/src/layers/archaic.rs @@ -0,0 +1,2 @@ +//! The archaic segments layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/layers/clinvar.rs b/rust/crates/og-map/src/layers/clinvar.rs new file mode 100644 index 000000000..cec5af38b --- /dev/null +++ b/rust/crates/og-map/src/layers/clinvar.rs @@ -0,0 +1,2 @@ +//! The ClinVar layer of the genome map. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-map/src/layers/mca.rs b/rust/crates/og-map/src/layers/mca.rs new file mode 100644 index 000000000..c94e144ba --- /dev/null +++ b/rust/crates/og-map/src/layers/mca.rs @@ -0,0 +1,2 @@ +//! The mosaic chromosome layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/layers/mod.rs b/rust/crates/og-map/src/layers/mod.rs index 29eeca48b..f19761991 100644 --- a/rust/crates/og-map/src/layers/mod.rs +++ b/rust/crates/og-map/src/layers/mod.rs @@ -1,3 +1,8 @@ //! One module per layer of the map. +pub mod archaic; +pub mod clinvar; +pub mod mca; +pub mod paint; pub mod roh; +pub mod sv; diff --git a/rust/crates/og-map/src/layers/paint.rs b/rust/crates/og-map/src/layers/paint.rs new file mode 100644 index 000000000..ac5b24a59 --- /dev/null +++ b/rust/crates/og-map/src/layers/paint.rs @@ -0,0 +1,2 @@ +//! The ancestry painting layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/layers/sv.rs b/rust/crates/og-map/src/layers/sv.rs new file mode 100644 index 000000000..1bc051f9e --- /dev/null +++ b/rust/crates/og-map/src/layers/sv.rs @@ -0,0 +1,2 @@ +//! The structural-variant layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/lib.rs b/rust/crates/og-map/src/lib.rs index 6769b8592..cc4643ef6 100644 --- a/rust/crates/og-map/src/lib.rs +++ b/rust/crates/og-map/src/lib.rs @@ -7,15 +7,20 @@ //! runs-of-homozygosity layer. The gene files (`genes`) are built once per //! GENCODE release, apart from any run. +pub mod crossovers; pub mod doc; pub mod facts; +pub mod family; pub mod findings; pub mod genes; pub mod layers; +pub mod live; pub mod predicate; pub mod reference; pub mod stamp; +pub mod svg; pub mod text; +pub mod threads; pub mod veil; use facts::{Facts, Sex}; diff --git a/rust/crates/og-map/src/live.rs b/rust/crates/og-map/src/live.rs new file mode 100644 index 000000000..cc371e081 --- /dev/null +++ b/rust/crates/og-map/src/live.rs @@ -0,0 +1,2 @@ +//! The genome map's answers to the live site. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/svg.rs b/rust/crates/og-map/src/svg.rs new file mode 100644 index 000000000..182b42da1 --- /dev/null +++ b/rust/crates/og-map/src/svg.rs @@ -0,0 +1,12 @@ +//! `og map svg`: the report's genome figure, drawn from the map model. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og map svg ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og map svg: not ported yet"); + 2 +} diff --git a/rust/crates/og-map/src/threads.rs b/rust/crates/og-map/src/threads.rs new file mode 100644 index 000000000..1c7a79568 --- /dev/null +++ b/rust/crates/og-map/src/threads.rs @@ -0,0 +1,2 @@ +//! The inherited threads of the family genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-pedigree/src/layout.rs b/rust/crates/og-pedigree/src/layout.rs new file mode 100644 index 000000000..d2bd31b80 --- /dev/null +++ b/rust/crates/og-pedigree/src/layout.rs @@ -0,0 +1,2 @@ +//! The drawing layout of a pedigree. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-pedigree/src/lib.rs b/rust/crates/og-pedigree/src/lib.rs index e05f68c02..16bcb0804 100644 --- a/rust/crates/og-pedigree/src/lib.rs +++ b/rust/crates/og-pedigree/src/lib.rs @@ -14,6 +14,10 @@ //! - [`Par`]: the pseudoautosomal regions of a BED file. //! //! A problem is a [`Die`]: `ERROR: ` lines on stderr and an exit code. +//! [`layout`], the drawing layout of a pedigree, is empty until its port +//! lands. + +pub mod layout; use std::collections::{HashMap, HashSet}; use std::fs::File; diff --git a/rust/crates/og-render/Cargo.toml b/rust/crates/og-render/Cargo.toml new file mode 100644 index 000000000..d785aae58 --- /dev/null +++ b/rust/crates/og-render/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-render" +description = "og report render|index|charts: the HTML and text reports, the run's search index and the chart specs, from summary.json" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-render/src/charts/mod.rs b/rust/crates/og-render/src/charts/mod.rs new file mode 100644 index 000000000..814b431b6 --- /dev/null +++ b/rust/crates/og-render/src/charts/mod.rs @@ -0,0 +1,13 @@ +//! `og report charts`: the chart specs the HTML report embeds and the live +//! site serves, which `bin/report_charts.py` builds today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og report charts ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og report charts: not ported yet"); + 2 +} diff --git a/rust/crates/og-render/src/index.rs b/rust/crates/og-render/src/index.rs new file mode 100644 index 000000000..4a9025aab --- /dev/null +++ b/rust/crates/og-render/src/index.rs @@ -0,0 +1,13 @@ +//! `og report index`: the search index of one run, every gene symbol and +//! rsID the summary names, which `bin/results_index.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og report index ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og report index: not ported yet"); + 2 +} diff --git a/rust/crates/og-render/src/lib.rs b/rust/crates/og-render/src/lib.rs new file mode 100644 index 000000000..c4d9f7301 --- /dev/null +++ b/rust/crates/og-render/src/lib.rs @@ -0,0 +1,13 @@ +//! The reports of one run, from its `summary.json`: the HTML and text +//! reports (`og report render`, [`render`]), the run's search index +//! (`og report index`, [`index`]), the chart specs (`og report charts`, +//! [`charts`]) and the report's sections, one module each ([`sections`]). +//! +//! Not ported yet: each verb says so and exits 2 until its unit of the +//! 1.0.0 plan lands. The crate exists now so those units add code without +//! editing the same files. + +pub mod charts; +pub mod index; +pub mod render; +pub mod sections; diff --git a/rust/crates/og-render/src/render.rs b/rust/crates/og-render/src/render.rs new file mode 100644 index 000000000..a0d37c7d5 --- /dev/null +++ b/rust/crates/og-render/src/render.rs @@ -0,0 +1,13 @@ +//! `og report render`: the HTML and text reports from summary.json, which +//! `bin/render_report.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og report render ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og report render: not ported yet"); + 2 +} diff --git a/rust/crates/og-render/src/sections/acmg_sf.rs b/rust/crates/og-render/src/sections/acmg_sf.rs new file mode 100644 index 000000000..70669f6e2 --- /dev/null +++ b/rust/crates/og-render/src/sections/acmg_sf.rs @@ -0,0 +1,2 @@ +//! The report section `acmg_sf`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/ancestry.rs b/rust/crates/og-render/src/sections/ancestry.rs new file mode 100644 index 000000000..f882faadb --- /dev/null +++ b/rust/crates/og-render/src/sections/ancestry.rs @@ -0,0 +1,2 @@ +//! The report section `ancestry`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/ancestry_fine.rs b/rust/crates/og-render/src/sections/ancestry_fine.rs new file mode 100644 index 000000000..d87e67915 --- /dev/null +++ b/rust/crates/og-render/src/sections/ancestry_fine.rs @@ -0,0 +1,2 @@ +//! The report section `ancestry_fine`. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-render/src/sections/ancient.rs b/rust/crates/og-render/src/sections/ancient.rs new file mode 100644 index 000000000..0bcd3409a --- /dev/null +++ b/rust/crates/og-render/src/sections/ancient.rs @@ -0,0 +1,2 @@ +//! The report section `ancient`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/annotsv.rs b/rust/crates/og-render/src/sections/annotsv.rs new file mode 100644 index 000000000..b9d5acde1 --- /dev/null +++ b/rust/crates/og-render/src/sections/annotsv.rs @@ -0,0 +1,2 @@ +//! The report section `annotsv`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/apoe.rs b/rust/crates/og-render/src/sections/apoe.rs new file mode 100644 index 000000000..de95c5197 --- /dev/null +++ b/rust/crates/og-render/src/sections/apoe.rs @@ -0,0 +1 @@ +//! The report section `apoe`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/archaic.rs b/rust/crates/og-render/src/sections/archaic.rs new file mode 100644 index 000000000..4295ec431 --- /dev/null +++ b/rust/crates/og-render/src/sections/archaic.rs @@ -0,0 +1,2 @@ +//! The report section `archaic`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/bloodtype.rs b/rust/crates/og-render/src/sections/bloodtype.rs new file mode 100644 index 000000000..afc1017a2 --- /dev/null +++ b/rust/crates/og-render/src/sections/bloodtype.rs @@ -0,0 +1,2 @@ +//! The report section `bloodtype`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/callability.rs b/rust/crates/og-render/src/sections/callability.rs new file mode 100644 index 000000000..b91918804 --- /dev/null +++ b/rust/crates/og-render/src/sections/callability.rs @@ -0,0 +1,2 @@ +//! The report section `callability`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/carrier.rs b/rust/crates/og-render/src/sections/carrier.rs new file mode 100644 index 000000000..98e47f503 --- /dev/null +++ b/rust/crates/og-render/src/sections/carrier.rs @@ -0,0 +1,2 @@ +//! The report section `carrier`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/cdna.rs b/rust/crates/og-render/src/sections/cdna.rs new file mode 100644 index 000000000..bd3715784 --- /dev/null +++ b/rust/crates/og-render/src/sections/cdna.rs @@ -0,0 +1 @@ +//! The report section `cdna`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/cdna_scan.rs b/rust/crates/og-render/src/sections/cdna_scan.rs new file mode 100644 index 000000000..722f7f3a5 --- /dev/null +++ b/rust/crates/og-render/src/sections/cdna_scan.rs @@ -0,0 +1,2 @@ +//! The report section `cdna_scan`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/chip.rs b/rust/crates/og-render/src/sections/chip.rs new file mode 100644 index 000000000..b84420658 --- /dev/null +++ b/rust/crates/og-render/src/sections/chip.rs @@ -0,0 +1 @@ +//! The report section `chip`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/clinvar.rs b/rust/crates/og-render/src/sections/clinvar.rs new file mode 100644 index 000000000..9876ec648 --- /dev/null +++ b/rust/crates/og-render/src/sections/clinvar.rs @@ -0,0 +1,2 @@ +//! The report section `clinvar`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/cn_intervals.rs b/rust/crates/og-render/src/sections/cn_intervals.rs new file mode 100644 index 000000000..11bb963e1 --- /dev/null +++ b/rust/crates/og-render/src/sections/cn_intervals.rs @@ -0,0 +1,2 @@ +//! The report section `cn_intervals`. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-render/src/sections/comphet.rs b/rust/crates/og-render/src/sections/comphet.rs new file mode 100644 index 000000000..df853c692 --- /dev/null +++ b/rust/crates/og-render/src/sections/comphet.rs @@ -0,0 +1,2 @@ +//! The report section `comphet`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/couple.rs b/rust/crates/og-render/src/sections/couple.rs new file mode 100644 index 000000000..cf30455af --- /dev/null +++ b/rust/crates/og-render/src/sections/couple.rs @@ -0,0 +1,2 @@ +//! The report section `couple`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/coverage.rs b/rust/crates/og-render/src/sections/coverage.rs new file mode 100644 index 000000000..f4023fc69 --- /dev/null +++ b/rust/crates/og-render/src/sections/coverage.rs @@ -0,0 +1,2 @@ +//! The report section `coverage`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/cpsr.rs b/rust/crates/og-render/src/sections/cpsr.rs new file mode 100644 index 000000000..4db6327f4 --- /dev/null +++ b/rust/crates/og-render/src/sections/cpsr.rs @@ -0,0 +1 @@ +//! The report section `cpsr`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/cyp21a2.rs b/rust/crates/og-render/src/sections/cyp21a2.rs new file mode 100644 index 000000000..3699dbddb --- /dev/null +++ b/rust/crates/og-render/src/sections/cyp21a2.rs @@ -0,0 +1,2 @@ +//! The report section `cyp21a2`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/deep_intronic.rs b/rust/crates/og-render/src/sections/deep_intronic.rs new file mode 100644 index 000000000..8bda20f8a --- /dev/null +++ b/rust/crates/og-render/src/sections/deep_intronic.rs @@ -0,0 +1,2 @@ +//! The report section `deep_intronic`. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-render/src/sections/dosage.rs b/rust/crates/og-render/src/sections/dosage.rs new file mode 100644 index 000000000..49d9dbfa1 --- /dev/null +++ b/rust/crates/og-render/src/sections/dosage.rs @@ -0,0 +1,2 @@ +//! The report section `dosage`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/exomiser.rs b/rust/crates/og-render/src/sections/exomiser.rs new file mode 100644 index 000000000..2a2b69ea8 --- /dev/null +++ b/rust/crates/og-render/src/sections/exomiser.rs @@ -0,0 +1,2 @@ +//! The report section `exomiser`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/g2.rs b/rust/crates/og-render/src/sections/g2.rs new file mode 100644 index 000000000..6f24ec5e0 --- /dev/null +++ b/rust/crates/og-render/src/sections/g2.rs @@ -0,0 +1 @@ +//! The report section `g2`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/haplogroups.rs b/rust/crates/og-render/src/sections/haplogroups.rs new file mode 100644 index 000000000..8f28f8f32 --- /dev/null +++ b/rust/crates/og-render/src/sections/haplogroups.rs @@ -0,0 +1,2 @@ +//! The report section `haplogroups`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/hba.rs b/rust/crates/og-render/src/sections/hba.rs new file mode 100644 index 000000000..d3f347f6a --- /dev/null +++ b/rust/crates/og-render/src/sections/hba.rs @@ -0,0 +1 @@ +//! The report section `hba`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/hla.rs b/rust/crates/og-render/src/sections/hla.rs new file mode 100644 index 000000000..79b3352a3 --- /dev/null +++ b/rust/crates/og-render/src/sections/hla.rs @@ -0,0 +1 @@ +//! The report section `hla`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/locityper.rs b/rust/crates/og-render/src/sections/locityper.rs new file mode 100644 index 000000000..1e0636177 --- /dev/null +++ b/rust/crates/og-render/src/sections/locityper.rs @@ -0,0 +1,2 @@ +//! The report section `locityper`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/lpa.rs b/rust/crates/og-render/src/sections/lpa.rs new file mode 100644 index 000000000..087b3e19b --- /dev/null +++ b/rust/crates/og-render/src/sections/lpa.rs @@ -0,0 +1 @@ +//! The report section `lpa`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/mei.rs b/rust/crates/og-render/src/sections/mei.rs new file mode 100644 index 000000000..f1c507eb1 --- /dev/null +++ b/rust/crates/og-render/src/sections/mei.rs @@ -0,0 +1 @@ +//! The report section `mei`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/mito.rs b/rust/crates/og-render/src/sections/mito.rs new file mode 100644 index 000000000..e6f32d585 --- /dev/null +++ b/rust/crates/og-render/src/sections/mito.rs @@ -0,0 +1 @@ +//! The report section `mito`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/mocha.rs b/rust/crates/og-render/src/sections/mocha.rs new file mode 100644 index 000000000..b56dd3a80 --- /dev/null +++ b/rust/crates/og-render/src/sections/mocha.rs @@ -0,0 +1,2 @@ +//! The report section `mocha`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/mod.rs b/rust/crates/og-render/src/sections/mod.rs new file mode 100644 index 000000000..8ce9d5696 --- /dev/null +++ b/rust/crates/og-render/src/sections/mod.rs @@ -0,0 +1,54 @@ +//! The report's sections, one module each, so the units that port them +//! never edit the same file. + +pub mod acmg_sf; +pub mod ancestry; +pub mod ancestry_fine; +pub mod ancient; +pub mod annotsv; +pub mod apoe; +pub mod archaic; +pub mod bloodtype; +pub mod callability; +pub mod carrier; +pub mod cdna; +pub mod cdna_scan; +pub mod chip; +pub mod clinvar; +pub mod cn_intervals; +pub mod comphet; +pub mod couple; +pub mod coverage; +pub mod cpsr; +pub mod cyp21a2; +pub mod deep_intronic; +pub mod dosage; +pub mod exomiser; +pub mod g2; +pub mod haplogroups; +pub mod hba; +pub mod hla; +pub mod locityper; +pub mod lpa; +pub mod mei; +pub mod mito; +pub mod mocha; +pub mod paralogs; +pub mod parascopy; +pub mod pgx; +pub mod prs; +pub mod qc; +pub mod rfc1; +pub mod roh; +pub mod run; +pub mod smn1; +pub mod str; +pub mod sv_callers; +pub mod telomere; +pub mod traits; +pub mod two_hits; +pub mod variants; +pub mod viral; +pub mod vntyper; +pub mod y_tree; +pub mod ystr; diff --git a/rust/crates/og-render/src/sections/paralogs.rs b/rust/crates/og-render/src/sections/paralogs.rs new file mode 100644 index 000000000..fffdfd985 --- /dev/null +++ b/rust/crates/og-render/src/sections/paralogs.rs @@ -0,0 +1,2 @@ +//! The report section `paralogs`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/parascopy.rs b/rust/crates/og-render/src/sections/parascopy.rs new file mode 100644 index 000000000..e688a9b01 --- /dev/null +++ b/rust/crates/og-render/src/sections/parascopy.rs @@ -0,0 +1,2 @@ +//! The report section `parascopy`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/pgx.rs b/rust/crates/og-render/src/sections/pgx.rs new file mode 100644 index 000000000..0591b1e47 --- /dev/null +++ b/rust/crates/og-render/src/sections/pgx.rs @@ -0,0 +1 @@ +//! The report section `pgx`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs new file mode 100644 index 000000000..3d832baa8 --- /dev/null +++ b/rust/crates/og-render/src/sections/prs.rs @@ -0,0 +1 @@ +//! The report section `prs`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/qc.rs b/rust/crates/og-render/src/sections/qc.rs new file mode 100644 index 000000000..3292a6ba8 --- /dev/null +++ b/rust/crates/og-render/src/sections/qc.rs @@ -0,0 +1 @@ +//! The report section `qc`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/rfc1.rs b/rust/crates/og-render/src/sections/rfc1.rs new file mode 100644 index 000000000..aff217dcf --- /dev/null +++ b/rust/crates/og-render/src/sections/rfc1.rs @@ -0,0 +1 @@ +//! The report section `rfc1`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/roh.rs b/rust/crates/og-render/src/sections/roh.rs new file mode 100644 index 000000000..4adf29175 --- /dev/null +++ b/rust/crates/og-render/src/sections/roh.rs @@ -0,0 +1 @@ +//! The report section `roh`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/run.rs b/rust/crates/og-render/src/sections/run.rs new file mode 100644 index 000000000..e23cc28cf --- /dev/null +++ b/rust/crates/og-render/src/sections/run.rs @@ -0,0 +1 @@ +//! The report section `run`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/smn1.rs b/rust/crates/og-render/src/sections/smn1.rs new file mode 100644 index 000000000..eb4742abe --- /dev/null +++ b/rust/crates/og-render/src/sections/smn1.rs @@ -0,0 +1 @@ +//! The report section `smn1`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/str.rs b/rust/crates/og-render/src/sections/str.rs new file mode 100644 index 000000000..047c968c6 --- /dev/null +++ b/rust/crates/og-render/src/sections/str.rs @@ -0,0 +1 @@ +//! The report section `str`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/sv_callers.rs b/rust/crates/og-render/src/sections/sv_callers.rs new file mode 100644 index 000000000..13956ab3f --- /dev/null +++ b/rust/crates/og-render/src/sections/sv_callers.rs @@ -0,0 +1,2 @@ +//! The report section `sv_callers`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/telomere.rs b/rust/crates/og-render/src/sections/telomere.rs new file mode 100644 index 000000000..0c43f4b3f --- /dev/null +++ b/rust/crates/og-render/src/sections/telomere.rs @@ -0,0 +1,2 @@ +//! The report section `telomere`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/traits.rs b/rust/crates/og-render/src/sections/traits.rs new file mode 100644 index 000000000..a087281ea --- /dev/null +++ b/rust/crates/og-render/src/sections/traits.rs @@ -0,0 +1,2 @@ +//! The report section `traits`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/two_hits.rs b/rust/crates/og-render/src/sections/two_hits.rs new file mode 100644 index 000000000..695939b43 --- /dev/null +++ b/rust/crates/og-render/src/sections/two_hits.rs @@ -0,0 +1,2 @@ +//! The report section `two_hits`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/variants.rs b/rust/crates/og-render/src/sections/variants.rs new file mode 100644 index 000000000..60ac2c9c6 --- /dev/null +++ b/rust/crates/og-render/src/sections/variants.rs @@ -0,0 +1,2 @@ +//! The report section `variants`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/viral.rs b/rust/crates/og-render/src/sections/viral.rs new file mode 100644 index 000000000..bb9ffa672 --- /dev/null +++ b/rust/crates/og-render/src/sections/viral.rs @@ -0,0 +1,2 @@ +//! The report section `viral`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/vntyper.rs b/rust/crates/og-render/src/sections/vntyper.rs new file mode 100644 index 000000000..d3a89d2e8 --- /dev/null +++ b/rust/crates/og-render/src/sections/vntyper.rs @@ -0,0 +1,2 @@ +//! The report section `vntyper`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/y_tree.rs b/rust/crates/og-render/src/sections/y_tree.rs new file mode 100644 index 000000000..892584a1c --- /dev/null +++ b/rust/crates/og-render/src/sections/y_tree.rs @@ -0,0 +1,2 @@ +//! The report section `y_tree`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/ystr.rs b/rust/crates/og-render/src/sections/ystr.rs new file mode 100644 index 000000000..7e7922e39 --- /dev/null +++ b/rust/crates/og-render/src/sections/ystr.rs @@ -0,0 +1 @@ +//! The report section `ystr`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-setup/src/lib.rs b/rust/crates/og-setup/src/lib.rs index 1a74193bb..f7fe0a98b 100644 --- a/rust/crates/og-setup/src/lib.rs +++ b/rust/crates/og-setup/src/lib.rs @@ -7,9 +7,12 @@ //! - [`genetic_map`] (`og setup genetic-map`): the PLINK GRCh38 genetic map //! per chromosome for steps 26b and 51, written atomically. //! -//! [`zip`] reads the archives the last two take. +//! [`zip`] reads the archives the last two take. [`revel`] (`og setup +//! revel`) and [`roh_af`] (`og setup roh-af`) are not ported yet. pub mod genetic_map; +pub mod revel; +pub mod roh_af; pub mod strchive; pub mod unzip; pub mod zip; diff --git a/rust/crates/og-setup/src/revel.rs b/rust/crates/og-setup/src/revel.rs new file mode 100644 index 000000000..114fb1cb8 --- /dev/null +++ b/rust/crates/og-setup/src/revel.rs @@ -0,0 +1,12 @@ +//! `og setup revel`: the REVEL GRCh38 score table that vcfanno reads. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og setup revel ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og setup revel: not ported yet"); + 2 +} diff --git a/rust/crates/og-setup/src/roh_af.rs b/rust/crates/og-setup/src/roh_af.rs new file mode 100644 index 000000000..21219064f --- /dev/null +++ b/rust/crates/og-setup/src/roh_af.rs @@ -0,0 +1,12 @@ +//! `og setup roh-af`: the allele-frequency panel that the ROH step reads. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og setup roh-af ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og setup roh-af: not ported yet"); + 2 +} diff --git a/rust/crates/og-steps/src/lib.rs b/rust/crates/og-steps/src/lib.rs index fae338970..2993b4349 100644 --- a/rust/crates/og-steps/src/lib.rs +++ b/rust/crates/og-steps/src/lib.rs @@ -17,10 +17,13 @@ //! check and its two tables. //! - [`sv`]: `og sv add-callers|postfilter`, step 05's SV calls into //! AnnotSV and the checks of its class 4 and 5 calls. +//! - [`longread`]: `og longread clair3-model`, the Clair3 model for the +//! basecaller (not ported yet). pub mod constraint; pub mod consumer; pub mod family; +pub mod longread; pub mod paraphase; pub mod pgx_rules; pub mod prs; diff --git a/rust/crates/og-steps/src/longread.rs b/rust/crates/og-steps/src/longread.rs new file mode 100644 index 000000000..f3dcb97bf --- /dev/null +++ b/rust/crates/og-steps/src/longread.rs @@ -0,0 +1,13 @@ +//! `og longread clair3-model`: the Clair3 model that matches the basecaller +//! of a long-read sample. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og longread clair3-model ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn clair3_model_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og longread clair3-model: not ported yet"); + 2 +} diff --git a/rust/crates/og-steps/src/sv.rs b/rust/crates/og-steps/src/sv.rs index 39ff04891..622dacdd9 100644 --- a/rust/crates/og-steps/src/sv.rs +++ b/rust/crates/og-steps/src/sv.rs @@ -15,7 +15,9 @@ //! ([`OverlapIndex`]). pub mod add_callers; +pub mod consensus_check; pub mod postfilter; +pub mod survivor_prep; use std::collections::{BTreeMap, HashMap}; diff --git a/rust/crates/og-steps/src/sv/consensus_check.rs b/rust/crates/og-steps/src/sv/consensus_check.rs new file mode 100644 index 000000000..a6c528d46 --- /dev/null +++ b/rust/crates/og-steps/src/sv/consensus_check.rs @@ -0,0 +1,13 @@ +//! `og sv consensus-check`: each record SURVIVOR merged checked for type, +//! strands and overlap, with its coordinates computed again. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og sv consensus-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og sv consensus-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-steps/src/sv/survivor_prep.rs b/rust/crates/og-steps/src/sv/survivor_prep.rs new file mode 100644 index 000000000..4f17375a3 --- /dev/null +++ b/rust/crates/og-steps/src/sv/survivor_prep.rs @@ -0,0 +1,13 @@ +//! `og sv survivor-prep`: one caller's structural variants made ready for +//! SURVIVOR's merge in step 22. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og sv survivor-prep ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og sv survivor-prep: not ported yet"); + 2 +} diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 4bbc9529f..2c87bff83 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -27,6 +27,7 @@ //! then `unreadable` with a note that names the file. pub mod acmg; +pub mod acmg_card; pub mod assets; pub mod carrier; pub mod clinical; diff --git a/rust/crates/og-summary/src/collect/acmg_card.rs b/rust/crates/og-summary/src/collect/acmg_card.rs new file mode 100644 index 000000000..9506a9040 --- /dev/null +++ b/rust/crates/og-summary/src/collect/acmg_card.rs @@ -0,0 +1,2 @@ +//! The summary section of the ACMG card, computed from the step 06 hits. +//! Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-vcf/src/lib.rs b/rust/crates/og-vcf/src/lib.rs index e8532bd5c..6f6994459 100644 --- a/rust/crates/og-vcf/src/lib.rs +++ b/rust/crates/og-vcf/src/lib.rs @@ -27,6 +27,7 @@ mod gt; mod header; pub mod rank; mod record; +pub mod shift; pub use csq::{CsqLayout, csq_field}; pub use gt::Gt; diff --git a/rust/crates/og-vcf/src/shift.rs b/rust/crates/og-vcf/src/shift.rs new file mode 100644 index 000000000..21f70ddf5 --- /dev/null +++ b/rust/crates/og-vcf/src/shift.rs @@ -0,0 +1,20 @@ +//! `og setup chrm-shift` and `og vcf shift-back`: the chrM reference +//! rotated by 8000 bp for the shifted-reference pass, and the calls on it +//! mapped back to chrM's own positions. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, each verb says +//! so on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og setup chrm-shift ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn chrm_shift_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og setup chrm-shift: not ported yet"); + 2 +} + +/// `og vcf shift-back ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn shift_back_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og vcf shift-back: not ported yet"); + 2 +} diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml index e401700b1..dc40567a0 100644 --- a/rust/og/Cargo.toml +++ b/rust/og/Cargo.toml @@ -30,6 +30,9 @@ og-summary.workspace = true og-lookup.workspace = true og-fresh.workspace = true og-recheck.workspace = true +og-render.workspace = true +og-live.workspace = true +og-couple.workspace = true [features] # BAM reading (og-bam, on a statically linked htslib written in C) is on by diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 7603ed025..c20fad8a1 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -25,16 +25,24 @@ fn tree_hash() -> &'static str { const USAGE: &str = "usage: og [args] og map [args] + og map svg [args] og run state|tasks|eta|headroom|delta [args] og recheck --sample S [--genome-dir DIR] [--offline FILE] [--rerun [--refresh-clinvar]] | --affected KEYS | --launch-dir DIR og setup mask-fasta SRC BED DEST og setup unzip ZIP DEST og setup genetic-map --zip ZIP --out DIR [--rfmix] CHROM... + og setup chrm-shift [args] + og setup revel [args] + og setup roh-af [args] og vcf csq-rank [options] + og vcf index [args] + og vcf shift-back [args] og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE og sv svlen-fix|inv-filter [args] og sv add-callers [args] og sv postfilter [args] + og sv survivor-prep [args] + og sv consensus-check [args] og pgx cpic-report|step36-outside [args] og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args] og pgx outside-calls|hla-concordance|pypgx-summary [args] @@ -51,6 +59,16 @@ const USAGE: &str = "usage: og [args] og prs table [args] og mito annotate|deletions|copies [args] og summary collect [--only {a,b,c}] [args] + og report render [args] + og report index [args] + og report charts [args] + og live serve [args] + og live results [args] + og live family [args] + og live demo [args] + og view serve [args] + og couple screen [args] + og couple genes [args] og roh summary --sample SAMPLE --roh ROH [--out OUT] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] @@ -62,7 +80,13 @@ const USAGE: &str = "usage: og [args] og family ibd [args] og consumer build-sites|export|check [args] og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report [args] + og ancestry-fine freqs [args] + og ancestry-fine project [args] + og ancestry-fine paint [args] + og ancestry-fine simulate [args] og archaic table|strict|genotypes|report [args] + og archaic compare [args] + og archaic genomes [args] og ancient select|lift|person-geno|align|report [args] og vntyper report [args] og locityper report [args] @@ -78,6 +102,7 @@ const USAGE: &str = "usage: og [args] og y yleaf-positions --data DATA OUT og exomiser prepare [args] og exomiser report [args] + og longread clair3-model [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] og dev scan personal-data|images [args] @@ -93,6 +118,17 @@ const USAGE: &str = "usage: og [args] og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args] og dev helpers [args] og dev build cn-intervals|callability|sv-rule-regions [args] + og dev build roh-af [args] + og dev build acmg-lof [args] + og dev demo-sample [args] + og dev demo-events [args] + og dev params-doc [args] + og dev python-gate [args] + og dev everything-check [args] + og dev licences [args] + og dev docs-check [args] + og dev stub-http [args] + og dev steps-check [args] og dev freshness [--out FILE] [--sections a,b] | --update-issue BODY --repo O/N | --self-test og version og --tree-hash @@ -105,6 +141,15 @@ fn usage_error(msg: &str) -> ExitCode { ExitCode::from(2) } +/// Runs a verb that writes to stdout and stderr; returns its exit code. +fn io(f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, args: &[String]) -> ExitCode { + ExitCode::from(f( + args, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )) +} + /// A BAM tool called on a build without the `bam` feature (the macOS build /// for a live host, rust/og.sh): it says so and exits 2, before any input /// is read, so a caller never mistakes it for a result. @@ -159,11 +204,15 @@ fn main() -> ExitCode { } } } + "map" if rest.first().is_some_and(|v| v == "svg") => io(og_map::svg::cli, &rest[1..]), "map" => map::run(rest, tree_hash()), "setup" => match rest.first().map(String::as_str) { Some("mask-fasta") => ExitCode::from(og_intervals::mask::cli(&rest[1..])), Some("unzip") => ExitCode::from(og_setup::unzip::cli(&rest[1..])), Some("genetic-map") => ExitCode::from(og_setup::genetic_map::cli(&rest[1..])), + Some("chrm-shift") => io(og_vcf::shift::chrm_shift_cli, &rest[1..]), + Some("revel") => io(og_setup::revel::cli, &rest[1..]), + Some("roh-af") => io(og_setup::roh_af::cli, &rest[1..]), Some(other) => usage_error(&format!("unknown setup tool '{other}'")), None => usage_error("the following arguments are required: setup tool"), }, @@ -181,6 +230,8 @@ fn main() -> ExitCode { "consumer" => ExitCode::from(og_steps::consumer::cli(rest)), "vcf" => match rest.first().map(String::as_str) { Some("csq-rank") => ExitCode::from(og_vcf::rank::cli(&rest[1..])), + Some("index") => io(og_io::tabix_write::cli, &rest[1..]), + Some("shift-back") => io(og_vcf::shift::shift_back_cli, &rest[1..]), Some(other) => usage_error(&format!("unknown vcf tool '{other}'")), None => usage_error("the following arguments are required: vcf tool"), }, @@ -194,6 +245,8 @@ fn main() -> ExitCode { Some("inv-filter") => ExitCode::from(og_filters::inv::cli(&rest[1..])), Some("add-callers") => ExitCode::from(og_steps::sv::add_callers::cli(&rest[1..])), Some("postfilter") => ExitCode::from(og_steps::sv::postfilter::cli(&rest[1..])), + Some("survivor-prep") => io(og_steps::sv::survivor_prep::cli, &rest[1..]), + Some("consensus-check") => io(og_steps::sv::consensus_check::cli, &rest[1..]), Some(other) => usage_error(&format!("unknown sv tool '{other}'")), None => usage_error("the following arguments are required: sv tool"), }, @@ -285,6 +338,12 @@ fn main() -> ExitCode { &mut std::io::stdout().lock(), &mut std::io::stderr().lock(), )), + "archaic" if rest.first().is_some_and(|v| v == "compare") => { + io(og_ancestry::compare::compare_cli, &rest[1..]) + } + "archaic" if rest.first().is_some_and(|v| v == "genomes") => { + io(og_ancestry::compare::genomes_cli, &rest[1..]) + } "archaic" => ExitCode::from(og_ancestry::archaic::cli( rest, &mut std::io::stdout().lock(), @@ -345,6 +404,37 @@ fn main() -> ExitCode { )), _ => usage_error("og summary takes collect"), }, + "report" => match rest.first().map(String::as_str) { + Some("render") => io(og_render::render::cli, &rest[1..]), + Some("index") => io(og_render::index::cli, &rest[1..]), + Some("charts") => io(og_render::charts::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown report tool '{other}'")), + None => usage_error("the following arguments are required: report tool"), + }, + "live" => match rest.first().map(String::as_str) { + Some("serve") => io(og_live::server::cli, &rest[1..]), + Some("results") => io(og_live::results::cli, &rest[1..]), + Some("family") => io(og_live::family::cli, &rest[1..]), + Some("demo") => io(og_live::demo::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown live tool '{other}'")), + None => usage_error("the following arguments are required: live tool"), + }, + "view" => match rest.first().map(String::as_str) { + Some("serve") => io(og_live::viewer::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown view tool '{other}'")), + None => usage_error("the following arguments are required: view tool"), + }, + "couple" => match rest.first().map(String::as_str) { + Some("screen") => io(og_couple::screen::cli, &rest[1..]), + Some("genes") => io(og_couple::genes::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown couple tool '{other}'")), + None => usage_error("the following arguments are required: couple tool"), + }, + "longread" => match rest.first().map(String::as_str) { + Some("clair3-model") => io(og_steps::longread::clair3_model_cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown longread tool '{other}'")), + None => usage_error("the following arguments are required: longread tool"), + }, "roh" => match rest.first().map(String::as_str) { Some("summary") => ExitCode::from(og_summary::collect::group_c::roh::cli( &rest[1..], diff --git a/rust/og/tests/port_scaffold.rs b/rust/og/tests/port_scaffold.rs new file mode 100644 index 000000000..da22c7cbc --- /dev/null +++ b/rust/og/tests/port_scaffold.rs @@ -0,0 +1,158 @@ +//! Every verb the 1.0.0 plan adds is registered in `og`: listed on its own +//! line of `og --help` and routed to its module, never to the "unknown" +//! error of a router. A port replaces a stub's body, so this test keeps +//! holding after it; a verb that a later merge drops turns it red. +//! +//! While a verb is still a stub it must say exactly `og : not ported +//! yet` on stderr and exit 2, so no caller mistakes it for a result. + +use std::io::Read; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Every verb the 1.0.0 plan adds, as its words after `og`. +const PLANNED: &[&str] = &[ + "report render", + "report index", + "report charts", + "live serve", + "live results", + "live family", + "live demo", + "view serve", + "couple screen", + "couple genes", + "vcf index", + "vcf shift-back", + "setup chrm-shift", + "setup revel", + "setup roh-af", + "sv survivor-prep", + "sv consensus-check", + "archaic compare", + "archaic genomes", + "ancestry-fine freqs", + "ancestry-fine project", + "ancestry-fine paint", + "ancestry-fine simulate", + "longread clair3-model", + "map svg", + "dev demo-sample", + "dev demo-events", + "dev params-doc", + "dev python-gate", + "dev everything-check", + "dev licences", + "dev docs-check", + "dev stub-http", + "dev steps-check", + "dev build roh-af", + "dev build acmg-lof", +]; + +/// An option no verb takes, so a ported verb stops at its argument check +/// instead of doing its work. +const PROBE: &str = "--og-scaffold-probe"; + +/// Runs `og ARGS` with no stdin; the exit code and stderr. A verb still +/// running after 30 s is killed and fails the test. +fn og(args: &[&str]) -> (Option, String) { + let mut child = Command::new(env!("CARGO_BIN_EXE_og")) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("og starts"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let reader = std::thread::spawn(move || { + let mut text = String::new(); + stderr.read_to_string(&mut text).map(|_| text) + }); + let start = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().expect("og waits") { + break status; + } + if start.elapsed() > Duration::from_secs(30) { + let _ = child.kill(); + let _ = child.wait(); + panic!("og {} {PROBE} still ran after 30 s", args.join(" ")); + } + std::thread::sleep(Duration::from_millis(20)); + }; + let text = reader.join().expect("reader").expect("utf-8 stderr"); + (status.code(), text) +} + +#[test] +fn every_planned_verb_has_its_own_usage_line() { + let out = Command::new(env!("CARGO_BIN_EXE_og")) + .arg("--help") + .output() + .expect("og --help runs"); + assert!(out.status.success()); + let usage = String::from_utf8(out.stdout).expect("utf-8 usage"); + let lines: Vec<&str> = usage + .lines() + .map(|l| l.trim_start_matches("usage:").trim()) + .collect(); + let missing: Vec<&&str> = PLANNED + .iter() + .filter(|verb| { + let want = format!("og {verb}"); + !lines + .iter() + .any(|l| l == &want || l.starts_with(&format!("{want} "))) + }) + .collect(); + assert!(missing.is_empty(), "no usage line for: {missing:?}"); +} + +#[test] +fn every_planned_verb_is_routed_and_a_stub_says_so() { + let mut failures = Vec::new(); + for verb in PLANNED { + let words: Vec<&str> = verb.split(' ').collect(); + let mut args = words.clone(); + args.push(PROBE); + let (code, stderr) = og(&args); + let last = words[words.len() - 1]; + let tool = words[words.len() - 2]; + let refusals = [ + format!("unknown tool '{tool}'"), + format!("unknown {tool} tool '{last}'"), + format!("unknown verb '{last}'"), + format!("unknown builder '{last}'"), + format!("invalid choice: '{last}'"), + ]; + if let Some(r) = refusals.iter().find(|r| stderr.contains(r.as_str())) { + failures.push(format!("og {verb}: not routed ({r})")); + } else if stderr.contains("not ported yet") { + let want = format!("og {verb}: not ported yet\n"); + if stderr != want || code != Some(2) { + failures.push(format!( + "og {verb}: a stub must print {want:?} and exit 2, got {stderr:?} and {code:?}" + )); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn an_unplanned_verb_is_still_refused() { + // The control for the test above: the router's refusal is what it looks for. + let (code, stderr) = og(&["sv", "no-such-verb", PROBE]); + assert_eq!(code, Some(2)); + assert!( + stderr.contains("unknown sv tool 'no-such-verb'"), + "{stderr}" + ); + let (code, stderr) = og(&["dev", "build", "no-such-builder", PROBE]); + assert_eq!(code, Some(2)); + assert!( + stderr.contains("unknown builder 'no-such-builder'"), + "{stderr}" + ); +} From abc792a15a375c7d08db6f2a0282d91932bef9c9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:22:03 +0100 Subject: [PATCH 52/86] fix(live): show viewer setup failures inside the reads panel Extend the synthetic reference to verify the 49 kb alignment window. Check camera changes, veil closure, Escape in the frame, and the original frame-src failure. Bead: pgp-iqon.11 --- bin/owngenome_live_viewer.py | 6 +- tests/test_live_viewer.py | 104 ++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/bin/owngenome_live_viewer.py b/bin/owngenome_live_viewer.py index e7891bb49..dddc65f25 100644 --- a/bin/owngenome_live_viewer.py +++ b/bin/owngenome_live_viewer.py @@ -15,8 +15,8 @@ (assets/viewer/serve.py sample_files): the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and the reference. A track whose file or index lies outside GENOME_DIR (a symlink out of it) is left out. The page and the files carry the -viewer's Content-Security-Policy (serve.CSP): the page talks to this site -only. The report the site serves (report.html) gets its viewer links pointed +viewer's Content-Security-Policy (serve.LIVE_CSP): the page talks to this site +only and allows same-origin framing. The report the site serves (report.html) gets its viewer links pointed here. Opening the page writes one access-log row; the byte ranges igv.js reads write none. @@ -137,7 +137,7 @@ def viewer(req, m): why = ready() if why: req.log("viewer", run, status=404) - return req.deny(404, why) + return send(req, 404, "text/plain; charset=utf-8", why) try: if page: with open(os.path.join(VIEWER_DIR, "index.html")) as f: diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index afe6ba730..803d7e33f 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -25,6 +25,9 @@ 7. in a headless Chrome (skipped without Chrome, except in CI) the page, loaded from the site, starts igv.js (status ready, 3 tracks) and the policy blocks nothing. + 8. the map opens a same-origin reads iframe, follows the camera, hides + closed findings and returns focus on Escape; a planted page policy + without frame-src blocks the iframe. Other routes keep DENY. Checks 2, 3 and 5 are also run against planted mistakes (a viewer that serves any file of the run folder, a report filter that does nothing, a viewer that compares the still-quoted name, one that serves an index out of GENOME_DIR) @@ -33,6 +36,7 @@ Run: python3 tests/test_live_viewer.py """ import http.client +import base64 import gzip import json import importlib.util @@ -197,7 +201,8 @@ def map_reads_checks(srv, work, chrome): """ app, port = srv[0].app, srv[1].server_address[1] policy = {"block": False} - page = ('' + page = ('' + '' '' '' '') @@ -221,20 +226,31 @@ def fixture(req, _): app.route("GET", r"/map-1g.json", lambda req, _: req.send_body(200, "application/gzip", data)) b = tl.Chrome(chrome, f"{work}/chrome-map-1g") try: + visit = 0 + + def js(expr): + reply = b.call("Runtime.evaluate", {"expression": expr, "returnByValue": True}, b.session) + if reply.get("exceptionDetails"): + raise RuntimeError(str(reply["exceptionDetails"])) + return reply.get("result", {}).get("value") + b.call("Emulation.setEmulatedMedia", {"features": [{"name": "prefers-color-scheme", "value": "dark"}, {"name": "prefers-reduced-motion", "value": "reduce"}]}, b.session) + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, b.session) def mount(demo=False): - b.go(f"http://127.0.0.1:{port}/map-1g/#chr1:1-49000") - if not b.wait("window.PGPMap && window.PGPUI"): + nonlocal visit + visit += 1 + b.go(f"http://127.0.0.1:{port}/map-1g/?visit={visit}#chr1:1-49000") + if not b.wait("!!(window.PGPMap && window.PGPUI)"): return False - b.eval("window.closedResult = true; " + js("window.closedResult = true;document.documentElement.dataset.run='S'; " "document.documentElement.dataset.mode = " + json.dumps("replay" if demo else "live") + "; " "window.testMap = PGPMap.create({url:'/map-1g.json'," "veil:function(id){return id==='closed' && window.closedResult ? 'closed':'open';}," "known:function(){return true;},now:function(){return true;}});" "document.body.appendChild(testMap.node);testMap.refresh('one');") - return b.wait("document.querySelector('.gmap[data-level=reads]')") + return b.wait("!!document.querySelector('.gmap[data-level=reads]')") def loaded(): return b.wait("(function(){var f=document.querySelector('.gm-reads iframe');" @@ -256,9 +272,21 @@ def loaded(): check("map: moving updates the same iframe to the map's locus", b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) check("viewer: alignments remain visible at 49 kb", - b.wait("(function(){var d=savedFrame.contentDocument;return d.querySelector('canvas.igv-canvas') && " - "!Array.from(d.querySelectorAll('.igv-zoom-in-notice-container')).some(function(x){return x.offsetHeight>0;});})()")) - b.eval("document.querySelector('.gm-reads-close').dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") + b.wait("(function(){var v=savedFrame.contentDocument.querySelector('.igv-viewport[data-track-type=alignment]');" + "var n=v && v.querySelector('.igv-zoom-in-notice-container');" + "return !!(v && v.querySelector('canvas') && n && n.style.display!=='flex');})()")) + if os.environ.get("MAP_SCREENS"): + os.makedirs(os.environ["MAP_SCREENS"], exist_ok=True) + for width, height in ((1440, 900), (390, 844)): + b.call("Emulation.setDeviceMetricsOverride", {"width": width, "height": height, + "deviceScaleFactor": 1, "mobile": False}, b.session) + b.eval("document.querySelector('.gm-reads').scrollIntoView({block:'center'})") + png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] + with open(os.path.join(os.environ["MAP_SCREENS"], f"map-reads-{width}.png"), "wb") as out: + out.write(base64.b64decode(png)) + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, + "deviceScaleFactor": 1, "mobile": False}, b.session) + b.eval("savedFrame.contentDocument.dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") check("map: Escape closes the reads and returns focus without moving the map", b.eval("document.querySelector('.gm-reads').hidden && document.activeElement===document.querySelector('.gm-canvas') && " "document.querySelector('.gmap').dataset.view==='chr1:1-49001'")) @@ -267,22 +295,39 @@ def loaded(): b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('[data-stop=reads]').click()") check("map: Reads gauge opens a 2 kb view and the panel", b.wait("!document.querySelector('.gm-reads').hidden && " "document.querySelector('.gmap').dataset.level==='reads'")) - b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('.gm-marks [data-id=open]').focus()") - b.eval("document.querySelector('.gm-card-reads').click()") + js("document.querySelector('.gm-reads-close').click();location.hash='chr1:1-2000'") + b.wait("document.querySelector('.gmap').dataset.view==='chr1:1-2000'") + js("document.querySelector('.gm-marks [data-id=open]').focus();document.querySelector('.gm-card-reads').click()") check("map: the card opens reads at its finding", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:401-501'")) - b.eval("window.closedResult=false;testMap.refresh('one');document.querySelector('.gm-marks [data-id=closed]').focus();" + js("window.closedResult=false;testMap.refresh('one');document.querySelector('.gm-marks [data-id=closed]').focus();" "document.querySelector('.gm-card-reads').click()") check("map: an opened finding offers reads", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:501-601'")) b.eval("window.closedResult=true;testMap.refresh('one')") check("map: closing a result removes its open reads panel and link", b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads')")) + # No finding in view: use the view itself, and no viewer flags: show the server's reason. + b.eval("location.hash='chr1:10000-59000'") + b.wait("document.querySelector('.gmap').dataset.view==='chr1:10000-59000'") + fasta = srv[0].cfg.viewer_fasta + srv[0].cfg.viewer_fasta = None + try: + b.eval("document.querySelector('.gm-reads-pill').click()") + check("map: a missing viewer shows the server's reason, without a broken frame", + b.wait("document.querySelector('.gm-reads').textContent.includes('not set up')") + and b.eval("!document.querySelector('.gm-reads iframe')")) + finally: + srv[0].cfg.viewer_fasta = fasta + # CI proves the browser assertion detects the original policy mistake. policy["block"] = True mount() b.eval("window.blockedFrames=0;document.addEventListener('securitypolicyviolation',function(e){" "if(e.violatedDirective==='frame-src')window.blockedFrames++;});document.querySelector('.gm-reads-pill').click()") - check("control: without frame-src the reads iframe is blocked", b.wait("window.blockedFrames>0") and not loaded()) + check("control: without frame-src the reads iframe is blocked", b.wait("window.blockedFrames>0") and b.eval( + "(function(){var f=document.querySelector('.gm-reads iframe');return !f || !f.contentDocument || " + "f.contentDocument.documentElement.dataset.pgpStatus!=='ready';})()"), + b.eval("document.documentElement.dataset.pgpBlocked")) policy["block"] = False mount(demo=True) b.eval("document.querySelector('.gm-reads-pill').click()") @@ -313,7 +358,19 @@ def main(): proc = None try: gd = f"{work}/genome" - tv.make_sample(gd) + _, contigs = tv.make_sample(gd) + # A 49 kb alignment check needs a reference longer than the original 3 kb sample. + contigs[0] = ("chr1", contigs[0][1] + "ACGT" * 14250) + fasta = f"{gd}/reference/synthetic.fasta" + with open(fasta, "w") as fa, open(fasta + ".fai", "w") as fai: + offset = 0 + for name, seq in contigs: + offset += len(name) + 2 + fai.write(f"{name}\t{len(seq)}\t{offset}\t60\t61\n") + fa.write(f">{name}\n" + "\n".join(seq[i:i + 60] for i in range(0, len(seq), 60)) + "\n") + offset += len(seq) + (len(seq) + 59) // 60 + tv.write_bam(f"{gd}/S/aligned/S_sorted.bam", contigs, + [(0, p, contigs[0][1][p:p + 50]) for p in range(400, 700, 10)]) s = f"{gd}/S" os.makedirs(f"{s}/nextflow") # the site lists a folder with nextflow/ as a run html, _ = tv.findings_report(work) @@ -402,18 +459,15 @@ def main(): if ci: FAILED.append("no Chrome in CI") else: - r = subprocess.run([chrome, "--headless=new", "--disable-gpu", "--no-sandbox", "--no-first-run", - "--no-default-browser-check", f"--user-data-dir={work}/chrome", - "--virtual-time-budget=20000", "--dump-dom", - f"http://127.0.0.1:{p}/runs/S/viewer/#chr1:451"], - capture_output=True, text=True, timeout=180) - status = re.search(r'data-pgp-status="([^"]*)"', r.stdout) - tracks = re.search(r'data-pgp-tracks="([^"]*)"', r.stdout) - blocked = re.search(r'data-pgp-blocked="([^"]*)"', r.stdout) - check("browser: the page from the site starts igv.js with the run's 3 tracks", - status and status.group(1) == "ready" and tracks and tracks.group(1) == "3", - (status and status.group(1), r.stderr[-800:], r.stdout[:600])) - check("browser: the policy blocked nothing the page needed", not blocked, blocked and blocked.group(1)) + b = tl.Chrome(chrome, f"{work}/chrome") + try: + b.go(f"http://127.0.0.1:{p}/runs/S/viewer/#chr1:451") + check("browser: the page from the site starts igv.js with the run's 3 tracks", + b.wait("document.documentElement.dataset.pgpStatus==='ready' && document.documentElement.dataset.pgpTracks==='3'"), + b.eval("document.documentElement.dataset.pgpStatus")) + check("browser: the policy blocked nothing the page needed", not b.eval("document.documentElement.dataset.pgpBlocked")) + finally: + b.close() map_reads_checks(srv, work, chrome) finally: stop(*srv) From 079b864e1488e163dbeeac90534c2a47af32c132 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:22:36 +0100 Subject: [PATCH 53/86] fix(pgx): stop warning against valproate for one recessive hit, and drop the SCN1A drug warnings One heterozygous pathogenic variant in ASL, ASS1, CPS1 or NAGS makes a carrier: the line is now a NOTE that says so and stays out of the summary. Both copies changed, or two variants in the gene, still warn. The rule lives in a new Report_if column of the contraindication table. POLG, OTC and HPRT1 keep warning on one variant. The SCN1A carbamazepine and phenytoin rows go: no label or guideline names the gene, as for ABL2. Each drug line prints its CPIC level as provisional, which every pair in the table is. The pgx-parse parity cases that read the table get goldens: the oracle cannot read the new column. Bead: pgx-21 Bead: pgx-22 Bead: pgx-23 Bead: D13 --- CHANGELOG.md | 2 + assets/pgx_contraindications.tsv | 40 +++++---- docs/27-cpic-lookup.md | 4 +- modules/local/cpic_lookup/main.nf | 6 +- rust/crates/og-pgx/src/cli.rs | 8 +- rust/crates/og-pgx/src/sections.rs | 86 ++++++++++++++++--- rust/crates/og-pgx/src/tables.rs | 52 +++++++++-- .../og-pgx/tests/i411_contraindications.rs | 11 ++- scripts/27-cpic-lookup.sh | 6 +- .../pgx-inhibitor-rows-1-contraindication.sh | 20 +++-- .../CONTRA_BAD_HITS_cpic_recommendations.txt | 37 ++++++++ .../golden/pgx-parse/contra-bad-hits/stderr | 1 + .../CONTRA_BAD_TABLE_cpic_recommendations.txt | 36 ++++++++ .../golden/pgx-parse/contra-bad-table/stderr | 1 + .../CONTRA_NO_HITS_cpic_recommendations.txt | 37 ++++++++ .../golden/pgx-parse/contra-no-hits/stderr | 0 .../out/CONTRA_NONE_cpic_recommendations.txt | 36 ++++++++ .../golden/pgx-parse/contra-none/stderr | 0 .../CONTRA_ON_LIST_cpic_recommendations.txt | 53 ++++++++++++ .../golden/pgx-parse/contra-on-list/stderr | 0 .../out/CONTRA_PLP_cpic_recommendations.txt | 46 ++++++++++ .../parity/golden/pgx-parse/contra-plp/stderr | 0 .../CONTRA_SHORT_ROW_cpic_recommendations.txt | 36 ++++++++ .../golden/pgx-parse/contra-short-row/stderr | 0 tests/test_cpic_parser.py | 17 ++-- 25 files changed, 472 insertions(+), 63 deletions(-) create mode 100644 tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-bad-hits/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-bad-table/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-no-hits/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-none/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-on-list/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-plp/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-short-row/stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..05209c768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed, or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/assets/pgx_contraindications.tsv b/assets/pgx_contraindications.tsv index f9c882a2f..0d6165987 100644 --- a/assets/pgx_contraindications.tsv +++ b/assets/pgx_contraindications.tsv @@ -5,24 +5,32 @@ # The rows are the CPIC gene-drug pairs at level A, B or A/B whose gene is a # disease gene outside PharmCAT and whose drug the variant makes risky # (CPIC API pair_view, queried 2026-11-03; Pair_source is the query for the -# gene). Left out: NAGS with carglumic acid (the treatment, not a risk), and -# ABL2 with valproic acid (no label or guideline names it). +# gene). Every pair here is provisional in CPIC (provisional true, no CPIC +# guideline): a gene with a CPIC guideline is one PharmCAT calls. A row needs +# a drug label that names the gene's disorder. Left out: NAGS with carglumic +# acid (the treatment, not a risk), ABL2 with valproic acid, and SCN1A with +# carbamazepine and phenytoin (no label or guideline names them; SCN1A also +# has gain-of-function disease variants, for which a warning against these +# drugs does not follow). # Gene: the HGNC symbol, as ClinVar's GENEINFO writes it. # Drug: the drug as CPIC names it. # Names: the first words a medication list may use for it, comma-separated, # lower case; a drug on the list is marked in the report. -# CPIC_level: the pair's CPIC level. -# Note: what the label says, or for a pair no label names, what CPIC says. -# Label_source: the FDA label (DailyMed) the note quotes, or '-' when none -# names the gene. +# CPIC_level: the pair's CPIC level, printed as provisional. +# Note: what the label says. +# Label_source: the FDA label (DailyMed) the note quotes. +# Report_if: when a hit gives a WARNING. any: one variant does (POLG, where +# one variant is a known valproate risk, and the X-linked OTC and HPRT1, +# where a male has one copy and a female carrier can be affected). +# biallelic: the disorder is recessive, so the WARNING needs both copies +# changed (a homozygous or hemizygous genotype) or two variants in the gene; +# one heterozygous variant makes a carrier and gives a NOTE instead. # tests/test_cpic_parser.py checks the shape of every row. -Gene Drug Names CPIC_level Note Pair_source Label_source -POLG valproic acid valproic,valproate,divalproex A/B The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.POLG https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -ASL valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASL https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -ASS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -CPS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.CPS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -NAGS valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.NAGS https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -OTC valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.OTC https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -HPRT1 mycophenolic acid mycophenolic,mycophenolate B The mycophenolate label says to avoid it in people with a hereditary HGPRT deficiency, such as Lesch-Nyhan or Kelley-Seegmiller syndrome. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.HPRT1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=37241e87-4af4-4dc3-a1aa-ea6f20d8dc40 -SCN1A carbamazepine carbamazepine B CPIC lists SCN1A with carbamazepine at level B, with no CPIC guideline and no label that names the gene: read its ClinPGx clinical annotations. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.SCN1A - -SCN1A phenytoin phenytoin B CPIC lists SCN1A with phenytoin at level B, with no CPIC guideline and no label that names the gene: read its ClinPGx clinical annotations. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.SCN1A - +Gene Drug Names CPIC_level Note Pair_source Label_source Report_if +POLG valproic acid valproic,valproate,divalproex A/B The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.POLG https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f any +ASL valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASL https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +ASS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +CPS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.CPS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +NAGS valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.NAGS https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +OTC valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.OTC https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f any +HPRT1 mycophenolic acid mycophenolic,mycophenolate B The mycophenolate label says to avoid it in people with a hereditary HGPRT deficiency, such as Lesch-Nyhan or Kelley-Seegmiller syndrome. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.HPRT1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=37241e87-4af4-4dc3-a1aa-ea6f20d8dc40 any diff --git a/docs/27-cpic-lookup.md b/docs/27-cpic-lookup.md index c65813c3f..180009f50 100644 --- a/docs/27-cpic-lookup.md +++ b/docs/27-cpic-lookup.md @@ -44,7 +44,7 @@ MEDICATIONS=/path/to/medications.txt ./scripts/27-cpic-lookup.sh your_name 5. With the pypgx summary it writes the comparison table and, for a gene PharmCAT reports as not called or ambiguous but pypgx called (CYP2D6 is the usual one), a warning in the recommendations naming every drug PharmCAT links to that gene. When pypgx's call is normal the line is a note instead. 6. With step 36's consensus table it writes a section "Calls From Other Tools": each gene passed to PharmCAT as an outside call, from which tool, and whether PharmCAT's report shows it as one (`callSource` `OUTSIDE`); and each gene held back, with the reason and what each caller said. For a held-back CYP2D6 it names the drugs CYP2D6 affects and says no guidance is given for them, and the pypgx-only warning of point 5 is not printed for it: one caller's call is what the consensus refused. Genes PharmCAT reports as outside calls carry `[outside call]` in the gene results. An outside call of CYP2D6, HLA-A or HLA-B in PharmCAT's report that the table does not pass on, with the same diplotype, comes from an earlier step 36 run that step 07 has not replaced: it gets no drug guidance, is listed as not called, and a warning says to rerun step 07. When the table is missing (a caller step ran again since) but PharmCAT's report has outside calls of those genes, the step passes the missing table, so none of them is confirmed. MT-RNR1's outside call comes from step 20's chrM calls through step 07, never from step 36: the table never lists it, so it is not checked against it and keeps its drug guidance. 7. With a medication list, each line is matched by its first word (a space, comma, semicolon, slash or plus ends a word, so `lopinavir/ritonavir` is matched as lopinavir; leading commas, semicolons, slashes and pluses are ignored, and a line of only those is skipped) against `assets/pgx_inhibitors.tsv`: the FDA's strong and moderate inhibitors of CYP2D6 and CYP2C19, its moderate CYP2C9 inhibitors (it lists no strong one) and its strong CYP3A inhibitors, one row each for CYP3A4 and CYP3A5, as the FDA's table of clinical CYP inhibitors classes them, each row with its source. A ritonavir-boosted combination the FDA classes as one (`lopinavir and ritonavir`) has a row under ritonavir and one under its partner. For CYP2D6 the step applies CPIC's rule from its opioid guideline: "for patients on strong CYP2D6 inhibitors, the CYP2D6 activity score is adjusted to 0 and the predicted phenotype is a poor metabolizer", and for a moderate inhibitor "the activity score is multiplied by 0.5 and then converted to the predicted phenotype" (CPIC's bands: 0 poor, under 1.25 intermediate, 1.25 to 2.25 normal, above 2.25 ultrarapid). The activity score is the one PharmCAT gives the called diplotype. With several inhibitors of one gene the strongest counts. The result is written as `Poor Metabolizer (phenoconverted by paroxetine)` beside the genotype phenotype, never in its place, with a `WARNING:` line in the section "Medications and Phenoconversion", so the summary carries it. Nothing is converted for a gene that is not called, ambiguous or unclassified, or for a moderate inhibitor when the report gives no activity score: the line says why. CPIC has no activity score for CYP2C19, and gives no rule that converts the phenotype of CYP2C9, CYP3A4 or CYP3A5, so an inhibitor of one of these genes gets a note and a `not computed: ...` value, with no converted phenotype. For a poor metabolizer of these genes (the most common CYP3A5 result) the note does not say the inhibitor can lower the activity further. A line that names no drug of the table is listed as "Not in the inhibitor table", never dropped. A list that names no drug converts nothing and leaves the phenotype table as it is without a list. An unreadable list or table is reported in that section; the rest of the report is written as usual. The FDA's table lists no strong or moderate CYP2B6 inhibitor, so CYP2B6 has no rows. -8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a `WARNING:` line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the CPIC level and the source; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call and whose drug the variant makes risky: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency) and SCN1A with carbamazepine and phenytoin (CPIC level B, no guideline or label names the gene). Without the hits table the section lists those genes as not checked. The line does not judge inheritance: one variant in a recessive gene usually makes a carrier, not a patient, so the section says to review it with a clinician. +8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, `1/2`, a hemizygous `1`) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. 9. A report that cannot be read, or that yields no gene, writes a "PARSING FAILED" report and the step exits 1. It never writes an all-clear report from a format it could not read. A field of the report with an unexpected type (a number where a name belongs, say) reads as missing. 10. The `Date:` line of the recommendations is today's date in UTC. The output files are written under a temporary name and renamed into place, so a step that stops half way leaves no half-written table. @@ -93,7 +93,7 @@ Only with a medication list. A drug can make you metabolize like a different phe ### Drug Contraindications From Disease Variants -A pathogenic variant in some disease genes makes a drug dangerous, and PharmCAT does not call those genes. This section checks step 06's ClinVar hits against `assets/pgx_contraindications.tsv`. A `WARNING:` line names the drug, the variant and its genotype, what the drug label says, and where to read it. Whether it applies to you depends on the disorder and on how many copies carry the variant, so read it with your clinician. Without step 06 the section says these genes were not checked; that is not an all-clear. +A pathogenic variant in some disease genes makes a drug dangerous, and PharmCAT does not call those genes. This section checks step 06's ClinVar hits against `assets/pgx_contraindications.tsv`. A `WARNING:` line names the drug, the variant and its genotype, what the drug label says, and where to read it. In a recessive gene one variant makes a carrier, not a patient: that gets a `NOTE:` line, not a warning. Whether a line applies to you depends on the disorder, so read it with your clinician. Without step 06 the section says these genes were not checked; that is not an all-clear. ### PharmCAT and pypgx diff --git a/modules/local/cpic_lookup/main.nf b/modules/local/cpic_lookup/main.nf index 1d9c4d6ca..962036931 100644 --- a/modules/local/cpic_lookup/main.nf +++ b/modules/local/cpic_lookup/main.nf @@ -20,8 +20,10 @@ or CYP3A5 gets a note. With CLINVAR_SCREEN's hits table and assets/pgx_contraindications.tsv, a pathogenic or likely pathogenic hit in one of its disease genes (POLG - and the urea-cycle genes with valproate, HPRT1, SCN1A) adds a WARNING - naming the drug; without the hits the report says they were not checked. + and the urea-cycle genes with valproate, HPRT1 with mycophenolate) adds + a WARNING naming the drug, or a NOTE for one heterozygous hit in a + recessive urea-cycle gene (a carrier); without the hits the report says + they were not checked. Fails when the report yields no gene. Equivalent to: scripts/27-cpic-lookup.sh diff --git a/rust/crates/og-pgx/src/cli.rs b/rust/crates/og-pgx/src/cli.rs index cd839bbef..580901eaa 100644 --- a/rust/crates/og-pgx/src/cli.rs +++ b/rust/crates/og-pgx/src/cli.rs @@ -355,9 +355,11 @@ fn contraindication_section(a: &CpicArgs, table_path: &str) -> Vec { out.extend(found); out.extend( [ - " Whether a warning applies depends on the disorder the gene causes and on how many", - " copies carry the variant: one copy in a recessive gene usually makes a carrier, not a", - " patient. Review it with your clinician before taking or stopping the drug.", + " In a recessive gene a WARNING needs both copies changed or two variants in the gene.", + " Two variants are not phased here: on the same copy they make a carrier. One variant", + " in a recessive gene gives a NOTE: a carrier, not a patient, unless a second variant", + " was missed. Whether a line applies depends on the disorder the gene causes.", + " Review it with your clinician before taking or stopping the drug.", "", ] .map(str::to_owned), diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index 4ec2173bf..f2e59a628 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -6,8 +6,8 @@ use crate::pyio::PyErr; use crate::report::{GeneCall, STEP36_GENES, position_read, text, truthy_field}; use crate::status::{Status, is_normal}; use crate::tables::{ - ClinvarHit, ConsensusRow, Contraindication, Inhibitors, Medication, PypgxCall, norm_drug, - pypgx_called, + ClinvarHit, ConsensusRow, Contraindication, Inhibitors, Medication, PypgxCall, ReportIf, + norm_drug, pypgx_called, }; use pycompat::fmt::general; use pycompat::json::Value; @@ -643,22 +643,52 @@ pub fn hit_genes(geneinfo: Option<&str>) -> BTreeSet { .collect() } -/// WARNING lines for each pathogenic or likely pathogenic hit in a gene of -/// the table, one per drug the table names for the gene. A drug whose names -/// hold a name of the medication list is marked as on it. +/// Every allele of the genotype is called and none is the reference: both +/// copies changed (`1/1`, `1|2`), or the one copy of a hemizygous call (`1`). +/// `0/1`, `1|0` and `./1` are one changed copy. +fn both_copies(genotype: &str) -> bool { + genotype + .split(['/', '|']) + .all(|a| !matches!(a.trim(), "" | "." | "0")) +} + +/// The lines for each pathogenic or likely pathogenic hit in a gene of the +/// table, one entry per drug the table names for the gene: a `WARNING:` +/// line, or a `NOTE:` line for one heterozygous variant in a gene whose +/// row needs both copies (`Report_if` biallelic), which makes a carrier. +/// A recessive gene warns when a genotype shows both copies changed or +/// when it holds two distinct variants; their phase is not known here, so +/// two variants on one copy still warn. A drug whose names hold a name of +/// the medication list is marked as on it. Only `WARNING:` lines reach the +/// summary. pub fn contraindication_lines( hits: &[ClinvarHit], table: &[Contraindication], med_names: &HashSet, ) -> Vec { - let mut lines = Vec::new(); - for h in hits { - if !is_plp(h.clnsig.as_deref()) { - continue; + let plp: Vec<(&ClinvarHit, BTreeSet)> = hits + .iter() + .filter(|h| is_plp(h.clnsig.as_deref())) + .map(|h| (h, hit_genes(h.geneinfo.as_deref()))) + .collect(); + // The distinct variants of each gene, for the two-variant rule. + let mut variants: HashMap<&str, HashSet<(&str, &str, &str, &str)>> = HashMap::new(); + for (h, genes) in &plp { + for gene in genes { + variants.entry(gene.as_str()).or_default().insert(( + h.chrom.as_str(), + h.pos.as_str(), + h.ref_.as_str(), + h.alt.as_str(), + )); } - for gene in hit_genes(h.geneinfo.as_deref()) { + } + let mut lines = Vec::new(); + for (h, genes) in &plp { + for gene in genes { let where_ = format!("{gene} {}:{} {}>{}", h.chrom, h.pos, h.ref_, h.alt); - for r in table.iter().filter(|r| r.gene == gene) { + let n = variants.get(gene.as_str()).map_or(0, HashSet::len); + for r in table.iter().filter(|r| &r.gene == gene) { let on_list = r .names .split(',') @@ -669,13 +699,31 @@ pub fn contraindication_lines( } else { "" }; - lines.push(format!( - " WARNING: {} -- {where_}, genotype {}, ClinVar {}{on}.", + let head = format!( + "{} -- {where_}, genotype {}, ClinVar {}", r.drug, h.genotype, h.clnsig.as_deref().unwrap_or("None") + ); + match r.report_if { + ReportIf::Any => lines.push(format!(" WARNING: {head}{on}.")), + ReportIf::Biallelic if both_copies(&h.genotype) => lines.push(format!( + " WARNING: {head}, both copies changed in this recessive gene{on}." + )), + ReportIf::Biallelic if n >= 2 => lines.push(format!( + " WARNING: {head}, {n} variants in {gene}, a recessive gene (phase not known){on}." + )), + ReportIf::Biallelic => { + lines.push(format!(" NOTE: {head}{on}.")); + lines.push(format!( + " One variant in {gene}, a recessive gene: a carrier, not a patient, unless a second variant was missed." + )); + } + } + lines.push(format!( + " {} (provisional CPIC level {}: no CPIC guideline covers this pair)", + r.note, r.cpic_level )); - lines.push(format!(" {} (CPIC level {})", r.note, r.cpic_level)); let source = if r.label_source == "-" { &r.pair_source } else { @@ -734,4 +782,14 @@ mod tests { let g: Vec = hit_genes(Some("POLG:5428| FOO:1|:3")).into_iter().collect(); assert_eq!(g, ["FOO", "POLG"]); } + + #[test] + fn genotypes_with_both_copies_changed() { + for gt in ["1/1", "1|1", "1/2", "2|1", "1"] { + assert!(both_copies(gt), "{gt}"); + } + for gt in ["0/1", "1|0", "./1", "1/.", "0", "", "./."] { + assert!(!both_copies(gt), "{gt}"); + } + } } diff --git a/rust/crates/og-pgx/src/tables.rs b/rust/crates/og-pgx/src/tables.rs index cc7305a4f..ef8455d7e 100644 --- a/rust/crates/og-pgx/src/tables.rs +++ b/rust/crates/og-pgx/src/tables.rs @@ -238,7 +238,7 @@ pub fn read_medications(path: &str) -> Result, PyErr> { // --- contraindications --------------------------------------------------- /// The columns of `assets/pgx_contraindications.tsv`, in order. -pub const CONTRA_COLUMNS: [&str; 7] = [ +pub const CONTRA_COLUMNS: [&str; 8] = [ "Gene", "Drug", "Names", @@ -246,8 +246,20 @@ pub const CONTRA_COLUMNS: [&str; 7] = [ "Note", "Pair_source", "Label_source", + "Report_if", ]; +/// When a pathogenic hit in a row's gene gives a WARNING (the table's +/// `Report_if` column, in the words of `assets/acmg_sf.tsv`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReportIf { + /// One variant is enough (`any`). + Any, + /// The disorder is recessive: both copies changed, or two variants in + /// the gene (`biallelic`). One heterozygous variant is a carrier. + Biallelic, +} + /// One rule of the contraindication table: a disease gene PharmCAT does not /// call, and a drug a pathogenic variant in it makes risky. #[derive(Debug, Clone, PartialEq, Eq)] @@ -260,11 +272,12 @@ pub struct Contraindication { pub note: String, pub pair_source: String, pub label_source: String, + pub report_if: ReportIf, } /// The rows of the contraindication table; lines starting with `#` are -/// comments. A missing column or an empty cell is an error, never a row -/// read as half a rule. +/// comments. A missing column, an empty cell or a `Report_if` other than +/// `any` or `biallelic` is an error, never a row read as half a rule. pub fn read_contraindications(path: &str) -> Result, PyErr> { let (header, rows) = dict_rows(&uncommented(path)?)?; if header.as_deref() != Some(&CONTRA_COLUMNS.map(str::to_owned)[..]) { @@ -288,6 +301,17 @@ pub fn read_contraindications(path: &str) -> Result, PyErr repr_row(r) ))); } + let report_if = match cells[7] { + "any" => ReportIf::Any, + "biallelic" => ReportIf::Biallelic, + other => { + return Err(PyErr::value(format!( + "{path}: Report_if is {}, not any or biallelic: {}", + repr(other), + repr_row(r) + ))); + } + }; let own = |i: usize| cells[i].to_owned(); Ok(Contraindication { gene: own(0), @@ -297,6 +321,7 @@ pub fn read_contraindications(path: &str) -> Result, PyErr note: own(4), pair_source: own(5), label_source: own(6), + report_if, }) }) .collect() @@ -391,10 +416,13 @@ mod tests { Some(0.0) ); let contra = read_contraindications(&format!("{root}pgx_contraindications.tsv")).unwrap(); + assert!(contra.iter().any(|c| c.gene == "POLG" + && c.drug == "valproic acid" + && c.report_if == ReportIf::Any)); assert!( contra .iter() - .any(|c| c.gene == "POLG" && c.drug == "valproic acid") + .any(|c| c.gene == "ASS1" && c.report_if == ReportIf::Biallelic) ); } @@ -428,17 +456,27 @@ mod tests { assert_eq!( read_contraindications(&p).unwrap_err().to_string(), format!( - "ValueError: {p}: the header is ['Gene', 'Drug'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source']" + "ValueError: {p}: the header is ['Gene', 'Drug'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source', 'Report_if']" ) ); let p = tmp( "contra2.tsv", - "Gene\tDrug\tNames\tCPIC_level\tNote\tPair_source\tLabel_source\nPOLG\tx\ty\n", + "Gene\tDrug\tNames\tCPIC_level\tNote\tPair_source\tLabel_source\tReport_if\nPOLG\tx\ty\n", + ); + assert_eq!( + read_contraindications(&p).unwrap_err().to_string(), + format!( + "ValueError: {p}: a row has an empty or missing cell: {{'Gene': 'POLG', 'Drug': 'x', 'Names': 'y', 'CPIC_level': None, 'Note': None, 'Pair_source': None, 'Label_source': None, 'Report_if': None}}" + ) + ); + let p = tmp( + "contra3.tsv", + "Gene\tDrug\tNames\tCPIC_level\tNote\tPair_source\tLabel_source\tReport_if\nASL\tv\tv\tB\tn\tp\tl\trecessive\n", ); assert_eq!( read_contraindications(&p).unwrap_err().to_string(), format!( - "ValueError: {p}: a row has an empty or missing cell: {{'Gene': 'POLG', 'Drug': 'x', 'Names': 'y', 'CPIC_level': None, 'Note': None, 'Pair_source': None, 'Label_source': None}}" + "ValueError: {p}: Report_if is 'recessive', not any or biallelic: {{'Gene': 'ASL', 'Drug': 'v', 'Names': 'v', 'CPIC_level': 'B', 'Note': 'n', 'Pair_source': 'p', 'Label_source': 'l', 'Report_if': 'recessive'}}" ) ); let p = tmp("hits.tsv", "nothing\there\n"); diff --git a/rust/crates/og-pgx/tests/i411_contraindications.rs b/rust/crates/og-pgx/tests/i411_contraindications.rs index 33ce246ba..3d31046d3 100644 --- a/rust/crates/og-pgx/tests/i411_contraindications.rs +++ b/rust/crates/og-pgx/tests/i411_contraindications.rs @@ -131,10 +131,13 @@ fn every_row_quotes_a_label() { fn the_cpic_level_reads_as_provisional() { let out = lines(&[hit("POLG", "89300000", "T", "0/1")]); assert!( - out.iter().any(|l| l.ends_with( - "(provisional CPIC level A/B: no CPIC guideline covers this pair)" - )), + out.iter().any( + |l| l.ends_with("(provisional CPIC level A/B: no CPIC guideline covers this pair)") + ), + "{out:#?}" + ); + assert!( + !out.iter().any(|l| l.ends_with("(CPIC level A/B)")), "{out:#?}" ); - assert!(!out.iter().any(|l| l.ends_with("(CPIC level A/B)")), "{out:#?}"); } diff --git a/scripts/27-cpic-lookup.sh b/scripts/27-cpic-lookup.sh index d880d4a53..4a7581891 100755 --- a/scripts/27-cpic-lookup.sh +++ b/scripts/27-cpic-lookup.sh @@ -23,9 +23,9 @@ # Disease-variant contraindications: with step 06's ClinVar hits # (clinvar/_clinvar_hits.tsv), a pathogenic or likely pathogenic hit # in a gene of assets/pgx_contraindications.tsv (POLG and the urea-cycle -# genes with valproate, HPRT1 with mycophenolate, SCN1A) adds a WARNING -# naming the drug. Without step 06 the report says those genes were not -# checked. +# genes with valproate, HPRT1 with mycophenolate) adds a WARNING naming the +# drug; one heterozygous hit in a recessive urea-cycle gene gives a NOTE (a +# carrier). Without step 06 the report says those genes were not checked. # # Requires: PharmCAT output from step 7. No internet connection needed. set -euo pipefail diff --git a/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh b/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh index 0fa7bcdd5..6ffec9a70 100755 --- a/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh +++ b/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh @@ -5,7 +5,9 @@ # - step 27 on a synthetic PharmCAT report and a synthetic step 06 hits # table with a pathogenic POLG hit: the report gives the valproic acid # WARNING, marked when valproate is on MEDICATIONS; fluconazole on the list -# gives CYP2C9 a note in the fifth column; +# gives CYP2C9 a note in the fifth column; one heterozygous ASS1 hit (a +# recessive urea-cycle gene) gives a carrier NOTE and no WARNING, and an +# SCN1A hit gives no drug line; # - the same sample without the hits table lists the genes as not checked # and gives no valproic acid line (the bash control); # - Nextflow with --tools pharmcat,clinvar,cpic on the HG002 VCF and a @@ -30,8 +32,12 @@ cat > "${G}/${S2}/pharmcat/${S2}.report.json" <<'JSON' "drugs": {}} JSON HITS="${G}/${S2}/clinvar/${S2}_clinvar_hits.tsv" -printf 'chrom\tpos\tref\talt\tgenotype\tclinvar_id\tgeneinfo\tclnsig\tclnrevstat\n' > "$HITS" -printf 'chr15\t89300000\tC\tT\t0/1\t9900001\tPOLG:5428\tPathogenic\tcriteria_provided,_single_submitter\n' >> "$HITS" +{ + printf 'chrom\tpos\tref\talt\tgenotype\tclinvar_id\tgeneinfo\tclnsig\tclnrevstat\n' + printf 'chr15\t89300000\tC\tT\t0/1\t9900001\tPOLG:5428\tPathogenic\tcriteria_provided,_single_submitter\n' + printf 'chr9\t130500000\tG\tA\t0/1\t9900002\tASS1:445\tLikely_pathogenic\tcriteria_provided,_single_submitter\n' + printf 'chr2\t166000000\tC\tT\t0/1\t9900003\tSCN1A:6323\tPathogenic\tcriteria_provided,_single_submitter\n' +} > "$HITS" MED="${CASE_TMP}/medications.txt" printf 'valproate 500 mg\nfluconazole\n' > "$MED" REC="${G}/${S2}/cpic/${S2}_cpic_recommendations.txt" @@ -43,8 +49,12 @@ check_step_exit 27-cpic-lookup.sh OUT=$(cat "$REC" 2>/dev/null) check "step 27: a pathogenic POLG hit gives the valproic acid WARNING, marked as on the list" \ has 'WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Pathogenic; ON THE MEDICATION LIST\.' "$OUT" -check "step 27: the line carries the label's note and its CPIC level" \ - has 'mitochondrial disorders caused by POLG mutations.*\(CPIC level A/B\)' "$OUT" +check "step 27: the line carries the label's note and its provisional CPIC level" \ + has 'mitochondrial disorders caused by POLG mutations.*\(provisional CPIC level A/B: no CPIC guideline covers this pair\)' "$OUT" +check "step 27: one heterozygous ASS1 hit gives the carrier NOTE" \ + has 'NOTE: valproic acid -- ASS1 chr9:130500000 G>A, genotype 0/1, ClinVar Likely_pathogenic; ON THE MEDICATION LIST\.' "$OUT" +check "step 27: one heterozygous ASS1 hit gives no WARNING" lacks 'WARNING: valproic acid -- ASS1' "$OUT" +check "step 27: an SCN1A hit gives no drug line" lacks 'SCN1A' "$OUT" check "step 27: the step's log names the hits table" grep -q 'ClinVar hits (step 06): .*_clinvar_hits\.tsv' "$STEP_LOG" check "step 27: fluconazole gives CYP2C9 a note in the fifth column" \ grep -q $'^CYP2C9\t\\*1/\\*1\tNormal Metabolizer\tnormal\tnot computed: fluconazole inhibits CYP2C9' "$PHENO" diff --git a/tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt new file mode 100644 index 000000000..d7b252989 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt @@ -0,0 +1,37 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_BAD_HITS +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-bad-hits.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + Could not read the ClinVar hits table (ValueError: contra-bad-hits-hits.tsv: no column alt, chrom, clnsig, geneinfo, genotype, pos, ref). + Not checked: POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. Run step 06 again, then this step. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-bad-hits/stderr b/tests/parity/golden/pgx-parse/contra-bad-hits/stderr new file mode 100644 index 000000000..224ee4856 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-hits/stderr @@ -0,0 +1 @@ +WARNING: could not read the ClinVar hits table (ValueError: contra-bad-hits-hits.tsv: no column alt, chrom, clnsig, geneinfo, genotype, pos, ref) diff --git a/tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt new file mode 100644 index 000000000..75e7e8c4c --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt @@ -0,0 +1,36 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_BAD_TABLE +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-bad-table.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + Could not read the contraindication table (ValueError: contraindications-bad.tsv: the header is ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source', 'Report_if']). + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-bad-table/stderr b/tests/parity/golden/pgx-parse/contra-bad-table/stderr new file mode 100644 index 000000000..b6a974ac0 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-table/stderr @@ -0,0 +1 @@ +WARNING: could not read the contraindication table (ValueError: contraindications-bad.tsv: the header is ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source', 'Report_if']) diff --git a/tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt new file mode 100644 index 000000000..fb7d6d100 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt @@ -0,0 +1,37 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_NO_HITS +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-no-hits.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + Not checked: no ClinVar hits table (step 06, or clinvar in --tools, did not run). + Genes it would check: POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-no-hits/stderr b/tests/parity/golden/pgx-parse/contra-no-hits/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt new file mode 100644 index 000000000..c53dffc7e --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt @@ -0,0 +1,36 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_NONE +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-none.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + No pathogenic or likely pathogenic ClinVar hit in POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-none/stderr b/tests/parity/golden/pgx-parse/contra-none/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt new file mode 100644 index 000000000..9fdd02c8f --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt @@ -0,0 +1,53 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_ON_LIST +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-on-list.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Medications and Phenoconversion (CYP2D6, CYP2C19, CYP2C9, CYP3A4, CYP3A5): +------------------------------------------------------------------------ + + Medication list: contra-on-list.txt (1 line). + Not in the inhibitor table: Valproate 500 mg. + These were not checked. A drug missing from the table (a brand name, a weak inhibitor) + can still change the activity of CYP2D6, CYP2C19, CYP2C9, CYP3A4, CYP3A5. + The genotype phenotype in the table above is unchanged. A phenoconverted phenotype + holds while the inhibitor is taken. This is an aid for your clinician, not advice. + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Likely_pathogenic; ON THE MEDICATION LIST. + The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. (provisional CPIC level A/B: no CPIC guideline covers this pair) + https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f + In a recessive gene a WARNING needs both copies changed or two variants in the gene. + Two variants are not phased here: on the same copy they make a carrier. One variant + in a recessive gene gives a NOTE: a carrier, not a patient, unless a second variant + was missed. Whether a line applies depends on the disorder the gene causes. + Review it with your clinician before taking or stopping the drug. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-on-list/stderr b/tests/parity/golden/pgx-parse/contra-on-list/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt new file mode 100644 index 000000000..e1a4a8e39 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt @@ -0,0 +1,46 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_PLP +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-plp.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Pathogenic. + The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. (provisional CPIC level A/B: no CPIC guideline covers this pair) + https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f + WARNING: valproic acid -- POLG chr15:166000000 C>T, genotype 1/1, ClinVar Pathogenic/Likely_pathogenic. + The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. (provisional CPIC level A/B: no CPIC guideline covers this pair) + https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f + In a recessive gene a WARNING needs both copies changed or two variants in the gene. + Two variants are not phased here: on the same copy they make a carrier. One variant + in a recessive gene gives a NOTE: a carrier, not a patient, unless a second variant + was missed. Whether a line applies depends on the disorder the gene causes. + Review it with your clinician before taking or stopping the drug. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-plp/stderr b/tests/parity/golden/pgx-parse/contra-plp/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt new file mode 100644 index 000000000..3fd5f84d0 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt @@ -0,0 +1,36 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_SHORT_ROW +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-short-row.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + No pathogenic or likely pathogenic ClinVar hit in POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-short-row/stderr b/tests/parity/golden/pgx-parse/contra-short-row/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_cpic_parser.py b/tests/test_cpic_parser.py index 2713ad6d0..9fc7d96f5 100644 --- a/tests/test_cpic_parser.py +++ b/tests/test_cpic_parser.py @@ -89,7 +89,7 @@ # The genes a medication list is checked for, and the contraindication # table's columns, as og reads them. PHENOCONVERSION_GENES = ("CYP2D6", "CYP2C19", "CYP2C9", "CYP3A4", "CYP3A5") -CONTRA_COLUMNS = ["Gene", "Drug", "Names", "CPIC_level", "Note", "Pair_source", "Label_source"] +CONTRA_COLUMNS = ["Gene", "Drug", "Names", "CPIC_level", "Note", "Pair_source", "Label_source", "Report_if"] def og(*args): @@ -374,20 +374,22 @@ def contraindication_checks(work): header = lines[0].rstrip("\n").split("\t") check("contraindications: the header", header == CONTRA_COLUMNS, header) rows = [line.rstrip("\n").split("\t") for line in lines[1:] if line.strip()] - bad = [r for r in rows if len(r) != 7 or not all(c.strip() for c in r) + bad = [r for r in rows if len(r) != 8 or not all(c.strip() for c in r) or r[3] not in ("A", "B", "A/B") or r[5] != f"https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.{r[0]}" - or not (r[6] == "-" or r[6].startswith("https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=")) + or not r[6].startswith("https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=") + or r[7] not in ("any", "biallelic") or any(n != n.strip().lower() or not n for n in r[2].split(",")) or r[1].split()[0] not in r[2].split(",")] check("contraindications: every row has a gene, a drug, its names, a CPIC level, a note and its sources", not bad, bad) keys = [(r[0], r[1]) for r in rows] check("contraindications: no gene-drug pair twice", len(keys) == len(set(keys)), keys) - check("contraindications: POLG with valproic acid, the urea-cycle genes, HPRT1 and SCN1A are there", + check("contraindications: POLG with valproic acid, the urea-cycle genes and HPRT1 are there", {("POLG", "valproic acid"), ("ASL", "valproic acid"), ("ASS1", "valproic acid"), ("CPS1", "valproic acid"), - ("NAGS", "valproic acid"), ("OTC", "valproic acid"), ("HPRT1", "mycophenolic acid"), - ("SCN1A", "carbamazepine"), ("SCN1A", "phenytoin")} <= set(keys), keys) + ("NAGS", "valproic acid"), ("OTC", "valproic acid"), ("HPRT1", "mycophenolic acid")} <= set(keys), keys) + check("contraindications: SCN1A is left out (no label or guideline names it)", + not any(r[0] == "SCN1A" for r in rows), keys) _, rec, _, _ = run_report(work, "CONTRAREAD", pheno_report(), contraindications=CONTRA) check("contraindications: the parser reads the table", "Could not read" not in contra_section(rec) and "Genes it would check: " + ", ".join(dict.fromkeys(r[0] for r in rows)) + "." in contra_section(rec), @@ -402,7 +404,8 @@ def contraindication_checks(work): check("contraindications: a pathogenic POLG hit adds the valproic acid WARNING", "WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Pathogenic." in sec, sec) check("contraindications: the line carries the label's note, the CPIC level and the label", - "mitochondrial disorders caused by POLG mutations" in sec and "(CPIC level A/B)" in sec + "mitochondrial disorders caused by POLG mutations" in sec + and "(provisional CPIC level A/B: no CPIC guideline covers this pair)" in sec and "dailymed.nlm.nih.gov" in sec, sec) check("contraindications: the section says to review it with a clinician", "Review it with your clinician" in sec, sec) From 4e4945ff5b2b378d0d10c4eeb37912e172f0e703 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:22:47 +0100 Subject: [PATCH 54/86] ci(guard): run every test with the og-leg marker instead of a hand-kept list Guard's rust job ran the tests that need og from two hand-kept lists, so every port edited the same lines. Each of those 41 tests now carries the line "# og-leg: OG_REQUIRED", and one step runs every tests/test_* file that has it. A planted marked test with a failing assert must be picked and fail first. A second step drops one planned verb's arm and the planned-verb test must fail. Bead: D1 Bead: D14 --- .github/workflows/guard.yml | 98 ++++++++++++++++------------- tests/test_ancestry_fine_report.py | 1 + tests/test_ancient_report.py | 1 + tests/test_annotsv_benign_mane.sh | 1 + tests/test_annotsv_postfilter.sh | 1 + tests/test_annotsv_sv_inputs.py | 1 + tests/test_annotsv_sv_inputs.sh | 1 + tests/test_archaic_report.py | 1 + tests/test_callability.py | 1 + tests/test_chip_report.py | 1 + tests/test_clinvar_join_adjacent.py | 1 + tests/test_collect_summary.py | 1 + tests/test_copy_number.py | 1 + tests/test_exomiser_report.py | 1 + tests/test_family_report.py | 1 + tests/test_family_segments.py | 1 + tests/test_hba_copy_number.py | 1 + tests/test_hla_second_caller.py | 1 + tests/test_ibd_report.py | 1 + tests/test_isec_columns.sh | 1 + tests/test_locityper_report.py | 1 + tests/test_lpa_kiv2.py | 1 + tests/test_mei_report.py | 1 + tests/test_mocha_report.py | 1 + tests/test_og_setup_zip.py | 1 + tests/test_paraphase_report.py | 1 + tests/test_pgx_mito_merge.py | 1 + tests/test_pgx_outside_calls.py | 1 + tests/test_pgx_refblock_fill.py | 1 + tests/test_render_report.py | 1 + tests/test_results_index.py | 1 + tests/test_run_delta.py | 1 + tests/test_setup_fine_maps.sh | 1 + tests/test_str_catalog.py | 1 + tests/test_str_ranges.py | 1 + tests/test_traits_report.py | 1 + tests/test_traits_table.py | 1 + tests/test_viral_report.py | 1 + tests/test_vntyper_report.py | 1 + tests/test_y_private_snps.py | 1 + tests/test_y_tree_place.py | 1 + tests/test_ystr_report.py | 1 + 42 files changed, 95 insertions(+), 44 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index a2aee4fac..f1a4c1821 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -374,6 +374,24 @@ jobs: - name: cargo test run: cargo test --workspace --locked --no-fail-fast + # The planned-verb test (og/tests/port_scaffold.rs) must fail when a + # later merge drops a verb's arm. + - name: The planned-verb test fails when a verb is dropped + run: | + cp og/src/main.rs "${RUNNER_TEMP}/main.rs" + trap 'cp "${RUNNER_TEMP}/main.rs" og/src/main.rs' EXIT + sed -i '/Some("survivor-prep") =>/d' og/src/main.rs + if grep -q 'survivor-prep") =>' og/src/main.rs; then echo "ERROR: the arm was not dropped"; exit 1; fi + log="${RUNNER_TEMP}/planned-verbs-red.log" + if cargo test --locked --no-fail-fast -p og --test port_scaffold > "$log" 2>&1; then + cat "$log" + echo "ERROR: the test passed with og sv survivor-prep dropped" + exit 1 + fi + cat "$log" + grep -F "og sv survivor-prep: not routed" "$log" + echo "Control: dropping og sv survivor-prep failed the planned-verb test." + # Each pinned test-only commit still contains the scanner it attacks. - name: Scanner controls (both rounds must fail against their pre-fix code) run: | @@ -571,19 +589,6 @@ jobs: file ../bin/.og/og-linux-* ../bin/og version - # Step 06 runs og clinvar in BCFTOOLS_IMAGE, so the tests that run the - # real step, or compare og clinvar with the summary, need bin/.og: they - # run here, and unit-tests skips them (no OG there). - - name: Step 06 with og (the tests that run og clinvar) - working-directory: . - env: - OG: ${{ github.workspace }}/bin/og - OG_REQUIRED: 1 - run: | - python3 tests/test_clinvar_join_adjacent.py - python3 tests/test_collect_summary.py - bash tests/test_isec_columns.sh - - name: The stale-binary guard refuses the real binaries once rust/ changes run: | echo "// edited" >> og/src/main.rs @@ -602,42 +607,47 @@ jobs: grep -q 'summary.json validates (og dev schema)' <<< "$out" \ || { echo "ERROR: reports-stale-step did not run og dev schema"; exit 1; } - # The tests of the og filters steps 04, 05 and 07 run, og chip's (step 49), og cn's (steps 35, 41 and 46), og family's (step 38), og run delta's, step 26b's og ancestry-fine, - # the opt-in step reports' (45, 47, 48, 50 to 53), step 37's Y tree tools, the - # og tools setup.sh runs (og str catalog, og setup unzip and genetic-map), and the step 25 - # tests that compare og prs format with the folders they write by hand: - # unit-tests has no og, so there they skip; here OG_REQUIRED turns a - # skip into a failure. The step 05 tests run the real step with bin/og. - - name: The og filter and setup tests, and step 05, with og + # Every test that carries the line `# og-leg: OG_REQUIRED` runs here, + # with bin/og and OG_REQUIRED=1 (the tests that run og clinvar, the + # og filters, setup and step report tools, and the step 05 tests, which + # run the real step with bin/og). unit-tests has no og, so there they + # skip with a note; here OG_REQUIRED turns a skip into a failure. A new + # test that needs og adds the marker line; there is no list to edit. + - name: The og-leg tests (every tests/test_* with the og-leg marker), with og working-directory: . env: OG: ${{ github.workspace }}/bin/og OG_REQUIRED: 1 run: | - fail=0 - for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py tests/test_run_delta.py \ - tests/test_ancestry_fine_report.py tests/test_archaic_report.py tests/test_ancient_report.py \ - tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ - tests/test_str_catalog.py tests/test_str_ranges.py tests/test_og_setup_zip.py tests/test_setup_fine_maps.sh \ - tests/test_render_report.py tests/test_collect_summary.py tests/test_chip_report.py \ - tests/test_family_report.py tests/test_family_segments.py tests/test_ibd_report.py \ - tests/test_vntyper_report.py tests/test_locityper_report.py tests/test_mocha_report.py \ - tests/test_mei_report.py tests/test_viral_report.py tests/test_ystr_report.py \ - tests/test_exomiser_report.py tests/test_y_tree_place.py tests/test_y_private_snps.py \ - tests/test_results_index.py \ - tests/test_pgx_outside_calls.py tests/test_hla_second_caller.py tests/test_paraphase_report.py \ - tests/test_copy_number.py tests/test_hba_copy_number.py \ - tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_callability.py; do - rc=0 - echo "::group::${t}" - case "$t" in - *.py) python3 "$t" || rc=$? ;; - *.sh) bash "$t" || rc=$? ;; - esac - echo "::endgroup::" - if [ "$rc" -eq 0 ]; then echo "PASS ${t}"; else echo "FAIL ${t} (exit ${rc})"; fail=1; fi - done - exit "$fail" + marker='# og-leg: OG_REQUIRED' + og_leg() { + local fail=0 t rc + for t in "$@"; do + rc=0 + echo "::group::${t}" + case "$t" in + *.py) python3 "$t" || rc=$? ;; + *.sh) bash "$t" || rc=$? ;; + esac + echo "::endgroup::" + if [ "$rc" -eq 0 ]; then echo "PASS ${t}"; else echo "FAIL ${t} (exit ${rc})"; fail=1; fi + done + return "$fail" + } + # Control: a planted test with the marker and a failing assert is + # picked by the glob and turns the leg red. + probe=tests/test_og_leg_probe.py + trap 'rm -f "$probe"' EXIT + printf '%s\nassert False, "planted"\n' "$marker" > "$probe" + picked=$(grep -lxF "$marker" tests/test_*.py tests/test_*.sh) + grep -qxF "$probe" <<< "$picked" || { echo "ERROR: the marker glob missed the planted test"; exit 1; } + if og_leg "$probe"; then echo "ERROR: the planted failing test passed"; exit 1; fi + rm -f "$probe" + echo "Control: the planted marked test was picked and failed." + mapfile -t tests < <(grep -lxF "$marker" tests/test_*.py tests/test_*.sh | sort) + [ "${#tests[@]}" -gt 0 ] || { echo "ERROR: no test carries the og-leg marker"; exit 1; } + echo "${#tests[@]} tests carry the og-leg marker" + og_leg "${tests[@]}" - name: Traits independent expectations reject changed reference labels and marker metadata working-directory: . diff --git a/tests/test_ancestry_fine_report.py b/tests/test_ancestry_fine_report.py index 1f00da38c..196c713bb 100755 --- a/tests/test_ancestry_fine_report.py +++ b/tests/test_ancestry_fine_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og ancestry-fine, step 26b's commands, on made-up inputs: panel-map a group of fewer than 10 unrelated people (one Oceanian, three diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py index b0e7efee9..aacc2535b 100644 --- a/tests/test_ancient_report.py +++ b/tests/test_ancient_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og ancient (step 44) on synthetic files, run as the pipeline runs it. Checks: diff --git a/tests/test_annotsv_benign_mane.sh b/tests/test_annotsv_benign_mane.sh index 00fadb8ce..88d1fe91f 100755 --- a/tests/test_annotsv_benign_mane.sh +++ b/tests/test_annotsv_benign_mane.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 05 (AnnotSV) with the two files setup.sh adds to AnnotSV's data, on a # made-up annotation set small enough for the unit-test job, with the real # AnnotSV image (ANNOTSV_IMAGE). diff --git a/tests/test_annotsv_postfilter.sh b/tests/test_annotsv_postfilter.sh index 2df9dd45b..861c61639 100755 --- a/tests/test_annotsv_postfilter.sh +++ b/tests/test_annotsv_postfilter.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # The class 4 and 5 checks with the real tools: step 05 (AnnotSV, then # og sv postfilter in ANNOTSV_IMAGE) and step 22 (SURVIVOR_IMAGE, # BCFTOOLS_IMAGE) on the made-up data of tests/fixtures/annotsv/class45.sh, diff --git a/tests/test_annotsv_sv_inputs.py b/tests/test_annotsv_sv_inputs.py index 04fda4a7a..e79c1b481 100755 --- a/tests/test_annotsv_sv_inputs.py +++ b/tests/test_annotsv_sv_inputs.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og sv inv-filter, og sv svlen-fix and og sv add-callers on made-up VCFs. The checks run og, the binary OG names (the rust job of guard.yml sets diff --git a/tests/test_annotsv_sv_inputs.sh b/tests/test_annotsv_sv_inputs.sh index b3ef6af48..c60a4cf27 100755 --- a/tests/test_annotsv_sv_inputs.sh +++ b/tests/test_annotsv_sv_inputs.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 05 (AnnotSV) on what it now reads: Manta's calls after step 04, with # the net length change of sequence-resolved records, plus the Delly and # CNVpytor calls no Manta call covers. The real AnnotSV image (ANNOTSV_IMAGE) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index 9fbeb2ea9..86e0fbf83 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. Checks: diff --git a/tests/test_callability.py b/tests/test_callability.py index 2fd1aace9..f085d245f 100644 --- a/tests/test_callability.py +++ b/tests/test_callability.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """The gene callability of step 16b and the ALT contig refusal. assets/callability_genes.tsv lists the genes short reads often cannot place diff --git a/tests/test_chip_report.py b/tests/test_chip_report.py index 42340069d..f68cb8608 100644 --- a/tests/test_chip_report.py +++ b/tests/test_chip_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """Step 49 (clonal haematopoiesis): og chip on synthetic bcftools output and on assets/chip_genes.tsv, and what the reports read of it. diff --git a/tests/test_clinvar_join_adjacent.py b/tests/test_clinvar_join_adjacent.py index aac315a6b..733fa4828 100755 --- a/tests/test_clinvar_join_adjacent.py +++ b/tests/test_clinvar_join_adjacent.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og clinvar join-adjacent, the combined-allele joiner of step 06 and the CLINVAR_SCREEN module, on hand-made records at real GRCh38 chr22 positions. diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 20eb59617..2a762dabe 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """bin/collect_summary.py and bin/render_report.py on synthetic sample folders. Checks: diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 04c53623c..c4f493616 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og cn and bin/cn_loci.tsv: one depth-ratio engine for every copy-number locus, compared with the pinned historical catalog reader and assess() oracle. diff --git a/tests/test_exomiser_report.py b/tests/test_exomiser_report.py index c13e52e96..e1f2a3a37 100644 --- a/tests/test_exomiser_report.py +++ b/tests/test_exomiser_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og exomiser prepare and og exomiser report: the symptom-driven ranking of step 48 (Exomiser). Every input is synthetic: a few-term OBO file, one- and four-sample VCF diff --git a/tests/test_family_report.py b/tests/test_family_report.py index 54d279915..f056e76b7 100644 --- a/tests/test_family_report.py +++ b/tests/test_family_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og family (step 38 and the Nextflow FAMILY workflow) on synthetic pedigrees, somalier tables and joint VCFs, and the pedigree rules other Python tools still import from bin/family_report.py. diff --git a/tests/test_family_segments.py b/tests/test_family_segments.py index 073e47e36..5efacacc2 100755 --- a/tests/test_family_segments.py +++ b/tests/test_family_segments.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og family segments (step 38's chromosome map) on a synthetic three-generation family: two grandparents, their son (the father), the mother, and a grandchild. Genotypes are drawn from known haplotypes, so the diff --git a/tests/test_hba_copy_number.py b/tests/test_hba_copy_number.py index 4dc92a38c..e9150c012 100755 --- a/tests/test_hba_copy_number.py +++ b/tests/test_hba_copy_number.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og cn call --locus HBA: step 35's HBA1/HBA2 count from short-read depth. It needs og: OG names the binary (OG=rust/target/release/og, or bin/og); without it the test is skipped, and with OG_REQUIRED set too it fails. diff --git a/tests/test_hla_second_caller.py b/tests/test_hla_second_caller.py index b978fde46..acde4f5bc 100644 --- a/tests/test_hla_second_caller.py +++ b/tests/test_hla_second_caller.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx outside-calls with OptiType, the opt-in second HLA caller. With HLA_QC=true (bash) or --hla_qc (Nextflow) step 08 also runs OptiType on diff --git a/tests/test_ibd_report.py b/tests/test_ibd_report.py index f3b39082d..25494cd6c 100644 --- a/tests/test_ibd_report.py +++ b/tests/test_ibd_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og family ibd: the DNA segments two sequenced relatives share (step 38). Each case writes synthetic Beagle VCFs, hap-ibd segment files, PLINK maps and diff --git a/tests/test_isec_columns.sh b/tests/test_isec_columns.sh index 599fd8dbd..7c23588ab 100755 --- a/tests/test_isec_columns.sh +++ b/tests/test_isec_columns.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 06 + both reports on a tiny synthetic genome. # # Builds a 200-base reference, a three-record sample VCF and a three-record diff --git a/tests/test_locityper_report.py b/tests/test_locityper_report.py index 122feb26a..3460d4615 100755 --- a/tests/test_locityper_report.py +++ b/tests/test_locityper_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og locityper report: step 53's table from what Locityper genotype wrote, and its section in both reports. diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index ae634adb0..e597f1bda 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og cn call --locus LPA_KIV2: step 46's LPA KIV-2 repeat count from short-read depth, with KIV-2 4925G>A beside it, and its Lp(a) section in both reports. Part 6 runs without og; the rest needs it: OG names the binary diff --git a/tests/test_mei_report.py b/tests/test_mei_report.py index 34873cdc9..71604d325 100644 --- a/tests/test_mei_report.py +++ b/tests/test_mei_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og mei report: the Alu, L1 and SVA insertions MEGAnE calls, and the known disease insertions (step 50). Each case writes a synthetic VCF in the format MEGAnE writes (the header and diff --git a/tests/test_mocha_report.py b/tests/test_mocha_report.py index 80aa58c86..469e2e0f5 100644 --- a/tests/test_mocha_report.py +++ b/tests/test_mocha_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og mocha report: step 51's mosaic chromosomal alterations from MoChA's tables. Each case writes MoChA's --stats and --calls tables and the per-chromosome diff --git a/tests/test_og_setup_zip.py b/tests/test_og_setup_zip.py index d15642265..557b0c653 100644 --- a/tests/test_og_setup_zip.py +++ b/tests/test_og_setup_zip.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og setup unzip and og setup genetic-map against the Python they replaced. scripts/setup.sh unpacked Exomiser's data zips with diff --git a/tests/test_paraphase_report.py b/tests/test_paraphase_report.py index 34758def9..f2f03d8cb 100755 --- a/tests/test_paraphase_report.py +++ b/tests/test_paraphase_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og paraphase: step 35b's read-group check and its two tables. Checks: diff --git a/tests/test_pgx_mito_merge.py b/tests/test_pgx_mito_merge.py index d2e64e324..6473c9e64 100644 --- a/tests/test_pgx_mito_merge.py +++ b/tests/test_pgx_mito_merge.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx mito-merge: the MT-RNR1 outside call PharmCAT gets from the chrM calls of step 20 / MITO_VARIANTS (step 07 and PHARMCAT run it). diff --git a/tests/test_pgx_outside_calls.py b/tests/test_pgx_outside_calls.py index 33664a9b1..579cb6cc1 100644 --- a/tests/test_pgx_outside_calls.py +++ b/tests/test_pgx_outside_calls.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx outside-calls: what reaches PharmCAT as an outside call. CYP2D6 reaches PharmCAT only when pypgx and a second caller (Cyrius) give the diff --git a/tests/test_pgx_refblock_fill.py b/tests/test_pgx_refblock_fill.py index 3f535a926..077cbedca 100644 --- a/tests/test_pgx_refblock_fill.py +++ b/tests/test_pgx_refblock_fill.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx refblock-fill: PharmCAT's multi-base positions called 0/0 from the reference blocks of a gVCF (steps 07 and PHARMCAT_PREPROCESS run it). diff --git a/tests/test_render_report.py b/tests/test_render_report.py index 36add70fe..d8ce8fdf8 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """The PRS path around pgsc_calc (og prs format and og prs table, then bin/collect_summary.py), and its card in bin/render_report.py, on synthetic files shaped like pgsc_calc's own. diff --git a/tests/test_results_index.py b/tests/test_results_index.py index db40a5348..c8c0b7839 100755 --- a/tests/test_results_index.py +++ b/tests/test_results_index.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """bin/results_index.py, its report appendix and the live search on a synthetic sample. The sample (tests/fixtures/results_index/SYNTH, invented) has ClinVar hits in diff --git a/tests/test_run_delta.py b/tests/test_run_delta.py index 97c4f9bca..173694ba8 100644 --- a/tests/test_run_delta.py +++ b/tests/test_run_delta.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og run delta on two synthetic runs of the same sample. Checks: diff --git a/tests/test_setup_fine_maps.sh b/tests/test_setup_fine_maps.sh index befc3a99a..a0d9d9bb9 100755 --- a/tests/test_setup_fine_maps.sh +++ b/tests/test_setup_fine_maps.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # setup.sh's install_fine_maps (step 26b's genetic maps) rebuilds a map an # older install left cut short. Before og, setup.sh skipped every chromosome # whose .rfmix.map was not empty, so a map a kill had cut short stayed diff --git a/tests/test_str_catalog.py b/tests/test_str_catalog.py index 4cfee57ba..051ceb690 100644 --- a/tests/test_str_catalog.py +++ b/tests/test_str_catalog.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og str catalog, and Stranger on the catalog it builds. Reads the two STRchive files versions.env pins (STRCHIVE_VERSION and the diff --git a/tests/test_str_ranges.py b/tests/test_str_ranges.py index 6482d4152..b842e9633 100644 --- a/tests/test_str_ranges.py +++ b/tests/test_str_ranges.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """The repeat ranges: the generated str_ranges.tsv, the report's cited subset and the docs. 1. str_ranges.tsv, which og str catalog writes beside the STRchive diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index 07fb6de3b..b1ad8e57b 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """Step 42 CLI goldens, rule-allele controls and og cn marker outputs. Internal validation, phase and bounded-reader assertions live in diff --git a/tests/test_traits_table.py b/tests/test_traits_table.py index 984d426f1..f9bdeb845 100644 --- a/tests/test_traits_table.py +++ b/tests/test_traits_table.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """Validate the curated traits catalog and its GRCh38 reference bases. Schema checks use og traits check. Rust unit tests plant invalid catalog diff --git a/tests/test_viral_report.py b/tests/test_viral_report.py index 7d16316a9..6ca3492e3 100644 --- a/tests/test_viral_report.py +++ b/tests/test_viral_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og viral report: the viral DNA in the reads that do not map to the human genome (step 47). Each case writes a synthetic viral FASTA, the samtools coverage and depth -aa diff --git a/tests/test_vntyper_report.py b/tests/test_vntyper_report.py index 8c1ac8384..c68b9b274 100644 --- a/tests/test_vntyper_report.py +++ b/tests/test_vntyper_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og vntyper report: step 52's reading of VNtyper's MUC1 VNTR result, and its section in both reports. diff --git a/tests/test_y_private_snps.py b/tests/test_y_private_snps.py index 5755f1617..dd93d7026 100755 --- a/tests/test_y_private_snps.py +++ b/tests/test_y_private_snps.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og y private-snps on a synthetic VCF and synthetic position tables (no real sample). diff --git a/tests/test_y_tree_place.py b/tests/test_y_tree_place.py index dbd385518..34691e828 100755 --- a/tests/test_y_tree_place.py +++ b/tests/test_y_tree_place.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og y tree-place and og y tree-positions on a synthetic tree, synthetic position tables and synthetic pileups (no real sample). diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py index 64e4815df..d8608e342 100644 --- a/tests/test_ystr_report.py +++ b/tests/test_ystr_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og y str-bed and og y str-report: Y-STR repeat counts from HipSTR's calls (step 45). Each case writes a synthetic HipSTR VCF (one sample, only the records it From 2b43c3c9d506601878e7e01d865bdb29daa5078a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:23:30 +0100 Subject: [PATCH 55/86] test(roh): pin scoped summaries and prove the controls fail Record ROH-only parity goldens and scope limits. Add the planted Guard control and migrate the retained report tests and missed e2e callers to the Rust collector contract. Keep the existing imprinting keys as neutral placeholders. Bead: lin-10 Bead: lin-13 Bead: lin-15 Bead: lin-16 Bead: lin-17 Bead: lic-41 --- .github/workflows/guard.yml | 31 + CHANGELOG.md | 4 + docs/11-roh-analysis.md | 36 +- .../og-summary/src/collect/group_c/roh.rs | 2 +- .../og-summary/src/collect/group_c/tests.rs | 6 + scripts/ci/e2e-case-paths.tsv | 4 +- tests/e2e/align-read-count-gate-1.sh | 4 +- tests/e2e/vendor-vcf-intake-2-nf-report.sh | 10 +- tests/parity/gen/summary-collect-c.sh | 7 +- .../summary-collect/all-bad/all-bad.json | 759 + .../all-history/all-history.json | 15569 ++++++++++++++++ .../golden/summary-collect/all-nf/all-nf.json | 14043 ++++++++++++++ .../golden/summary-collect/all/all.json | 15495 +++++++++++++++ .../golden/summary-collect/c-bad1/bad1.json | 762 + .../golden/summary-collect/c-full/full.json | 2402 +++ .../golden/summary-collect/c-nf/nf.json | 1200 ++ tests/test_collect_summary.py | 40 +- 17 files changed, 50326 insertions(+), 48 deletions(-) create mode 100644 tests/parity/golden/summary-collect/all-bad/all-bad.json create mode 100644 tests/parity/golden/summary-collect/all-history/all-history.json create mode 100644 tests/parity/golden/summary-collect/all-nf/all-nf.json create mode 100644 tests/parity/golden/summary-collect/all/all.json create mode 100644 tests/parity/golden/summary-collect/c-bad1/bad1.json create mode 100644 tests/parity/golden/summary-collect/c-full/full.json create mode 100644 tests/parity/golden/summary-collect/c-nf/nf.json diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index a2aee4fac..a77130baa 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -371,6 +371,37 @@ jobs: cp -f "$saved" "$src" trap - EXIT + - name: Control (ROH scope and neutral hints reject planted faults) + run: | + reader=crates/og-summary/src/collect/group_c/roh.rs + table=../assets/imprinted_regions.tsv + cp "$reader" "$RUNNER_TEMP/roh-reader.rs" + cp "$table" "$RUNNER_TEMP/imprinted-regions.tsv" + restore_roh() { + cp "$RUNNER_TEMP/roh-reader.rs" "$reader" + cp "$RUNNER_TEMP/imprinted-regions.tsv" "$table" + } + trap restore_roh EXIT + sed -i 's/let denominator = covered_unmasked_bp(&covered);/let denominator = autosome_unmasked_bp();/' "$reader" + sed -i 's/s("-")/s("Example syndrome")/g' "$reader" + sed -i 's/26248010;29959430/not-pmid/' "$table" + log="$RUNNER_TEMP/roh-scope-planted.log" + rc=0 + cargo test --locked -p og-summary --no-fail-fast > "$log" 2>&1 || rc=$? + cat "$log" + test "$rc" -ne 0 + for name in \ + roh_scope_chr20_uses_its_own_unmasked_denominator \ + roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty \ + roh_scope_a_segment_only_file_never_establishes_scope \ + roh_scope_upd_is_a_region_hint_without_syndrome_names \ + imprinted_asset_is_the_only_table; do + grep -E "test .*${name} .* FAILED" "$log" + done + grep -E 'test result: FAILED\. [0-9]+ passed; 5 failed;' "$log" + restore_roh + trap - EXIT + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cac8762..78de0ffed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- ROH summaries now divide F_ROH by the GRCh38 length outside the bands of the assessed autosomes. Per-site rows identify those chromosomes, including chromosomes with no ROH. A new `roh.data.scope` records the chromosomes, denominator and scope source. Relationship labels and UPD flags require sites on all 22 autosomes and the full unmasked autosome length. Segment-only files use their observed chromosomes as an unverified lower bound and get neither label. Empty input has null F_ROH. Segment counts and the step 11 summary table keep their format. +- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. +- The ROH collector now streams per-site rows instead of keeping the whole file in memory. On 1.1 million synthetic site rows, peak RSS fell from 65.7-67.0 MB to 3.0-3.5 MB. CPU time stayed near 0.2 seconds. Under shared host load, wall time ranged from 1.40-21.46 seconds before and 7.89-38.16 seconds after; these timings do not establish a speedup. + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 20a8146bb..07fc1e338 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -42,7 +42,7 @@ For chip data (no `FORMAT/PL`), the script adds `-G30`. ### Centromere and heterochromatin bands -Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `og summary collect` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. +Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10). Neighbouring rows are merged around each centromere, with two spans on chrY. `MASKED_BANDS` in `rust/crates/og-summary/src/collect/group_c/roh.rs` is their only copy. The step and the Nextflow module call `og roh summary`, which reads that same constant. On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. The reports count autosomes only: Segments, Total and Largest keep chr1 to chr22, so they leave out chrX and chrY (a man's single X reads as one long run) and any other contig (chrM, chrEBV, chrUn_*), and the part of each segment in a band. They say how many chrX and chrY segments and how many megabases in bands were left out. @@ -63,8 +63,15 @@ When parents are related, a child is autozygous (both copies from the same ances These are averages. Inheritance is random, so one person's total can be well above or below the value for their parents' relationship, and the ranges of neighbouring rows overlap. A total near one row is consistent with it; it does not prove it. -Both reports print this as **F_ROH**, the measure of McQuillan et al. 2008 (PMID 18760389): the length of the autosomal segments of 5 Mb or more, without their band parts, divided by the length of chr1 to chr22 without the bands (2,875,001,522 bp less 217,000,000 bp, so 2,658,001,522 bp). Beside it they name the row of the table above whose F is nearest to F_ROH on a log scale: the boundaries are the geometric means of neighbouring rows (for example 1/16 and 1/8 meet at about 0.088), and an F_ROH under 1/128 (about 21 Mb) reads "no sign of related parents". When one chromosome carries most of the runs (next section), there is no band: that chromosome fills F_ROH by itself. +`og summary collect` writes **F_ROH**, the measure of McQuillan et al. 2008 ([PMID 18760389](https://pubmed.ncbi.nlm.nih.gov/18760389/)). Its numerator is the length of autosomal runs of 5 Mb or more outside the bands. Its denominator is the GRCh38 length of the chromosomes this run assessed, with their bands removed. With all 22 autosomes, that is 2,875,001,522 bp less 217,000,000 bp, or 2,658,001,522 bp. With chr20 alone it is 64,444,167 bp less 4,700,000 bp, or 59,744,167 bp. A 21 Mb run then gives F_ROH 0.3515, rather than 0.0079. +The collector streams the per-site `ST` rows to identify assessed autosomes. A chromosome with sites counts even when it has no ROH. No marker-count floor is imposed, so sparse exome and chip input remain supported. This is a chromosome-length denominator, **not a count of callable bases**. One site does not establish coverage across a chromosome. A capture panel or a small interval on every autosome can still look complete at this resolution. + +The new `roh.data.scope` object records `autosomes` in numeric order, `unmasked_bp`, `source`, `partial` and a note. Older segment-only files have `source: segments_only`; their observed chromosomes give only a lower bound on scope. An empty file has no denominator and `f_roh: null`. The segment list and length totals remain available. + +A relationship label or a one-chromosome UPD flag needs sites on all 22 autosomes and the full 2,658,001,522 bp denominator. Below that floor, or without per-site evidence, `relationship` and `upd_flag` are null. A subset F_ROH is a descriptive fraction of the listed chromosomes and must not be mapped to the parental relationship table. The report and live-view wording for this scope is handled separately. + +For complete chromosome scope, the relationship is the row whose F is nearest on a log scale. Boundaries are the geometric means of neighbouring rows; for example 1/16 and 1/8 meet at about 0.088. An F_ROH under 1/128 reads "no sign of related parents". A one-chromosome flag suppresses that relationship label. To compute the total the table means from the summary file, which takes the band parts out before it keeps the segments of 5 Mb or more, as the reports do: ```bash @@ -78,24 +85,19 @@ In uniparental disomy (UPD) both copies of a chromosome come from one parent. Wh The reports flag a chromosome when its segments of 5 Mb or more (band parts left out) add up to at least 20 Mb **and** to more than half of the length of all such autosomal segments. Both numbers are our thresholds, not a published cut-off: 20 Mb on one chromosome is the long-standing sign of isodisomy (see the uniparental disomy note under [Important Notes](#important-notes)), and the share rule keeps out a child of related parents, whose runs spread over many chromosomes. The flag says "may": a run that long can also be chance in a child of related parents. -A flagged chromosome is looked up in `assets/imprinted_regions.tsv`, the chromosomes where UPD from one parent causes an imprinting disorder (overview: Eggermann et al. 2023, Nat Rev Dis Primers, PMID 37386011). A row with a region applies only when a run on that chromosome overlaps it; the others apply to the whole chromosome. +A flagged chromosome is looked up in [assets/imprinted_regions.tsv](../assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. -| Chromosome or region | UPD from the mother | UPD from the father | Source (PMID) | -|---|---|---|---| -| chr6 (6q24, PLAGL1, HYMAI) | none known | Transient neonatal diabetes mellitus | 20301706 | -| chr7 (GRB10 at 7p12, MEST at 7q32) | Silver-Russell syndrome | none known | 27585961 | -| 11p15, chr11:0-22,000,000 (H19/IGF2, KCNQ1OT1) | Silver-Russell syndrome | Beckwith-Wiedemann syndrome | 29377879, 27585961 | -| chr14 (14q32.2, DLK1-MEG3) | Temple syndrome | Kagami-Ogata syndrome | 24891339, 26377239 | -| 15q11-q13, chr15:20,500,000-33,400,000 (SNRPN, UBE3A) | Prader-Willi syndrome | Angelman syndrome | 22237428, 27615419 | -| chr20 (20q13.32, GNAS) | Mulchandani-Bhoj-Conlin syndrome (growth failure) | Pseudohypoparathyroidism type 1B | 26248010, 29959430 | +The flag lists the chromosome, run length, share, region names and citations. It names no syndrome. The existing `maternal_upd` and `paternal_upd` JSON keys stay as `"-"` for reader compatibility. The asset retains their reference facts. Those facts cannot be inferred from a region match in one genome. -`og summary collect` holds the same rows (`IMPRINTED_REGIONS` in `rust/crates/og-summary`; a unit test checks that the two agree). A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. +All six chromosomes use the same chromosome-wide lookup, including chr11 and chr15. Mixed iso/heterodisomy can leave an imprinted locus outside the observed homozygous runs ([Robinson 2000](https://pubmed.ncbi.nlm.nih.gov/10797485/)). A region hint therefore does not claim that a run overlaps the imprinted locus. Region names are broad cytoband labels, not diagnostic locus intervals or a specialist-reviewed assay. No interval was narrowed on the basis of a ROH alone. -What this cannot tell, and the reports say so: +The flag's `note` describes long chromosome-focused ROH and suggests investigating possible isodisomy. Its limits are: -- **Which parent.** A single genome has no parent of origin; both columns are listed. The parents' genotypes (step 38, the family workflow) confirm a UPD and tell which. Where a row applies, a methylation test of its imprinted region tests for that imprinting disorder; on its own it does not prove a UPD, since an imprinting defect gives the same result. -- **Heterodisomy.** Two different copies from one parent leave no run at all, so it is not detected here. It needs the parents' genotypes. -- **Mosaic or segmental UPD** that covers only a short stretch, or a share of the cells (common in Beckwith-Wiedemann syndrome), may stay under the thresholds. +- A large deletion of one copy or acquired loss of heterozygosity can give the same pattern. Check the copy-number results and the MoChA results. The collector does not establish copy neutrality or perform that cross-check. +- Shared ancestry, including related parents, can also produce long runs. A flag is a screening hint, not a UPD diagnosis. +- A single genome cannot determine parental origin. The parents' genotypes can establish it. Methylation testing assesses an imprinting defect and alone does not establish UPD. +- Heterodisomy can leave no ROH. Mosaic or short segmental UPD can fall below the thresholds. +- Sites on all autosomes establish chromosome presence only. They do not prove assay completeness within any chromosome. ### Single segments @@ -112,5 +114,5 @@ What this cannot tell, and the reports say so: - bcftools roh reads a site's frequency from INFO/AC and INFO/AN when the record has them, and uses 0.4 only where it does not. DeepVariant writes neither, so on its calls every site uses 0.4. A VCF from another caller may carry them: in a single-sample file they are the sample's own allele counts, not a population's. - No population allele frequencies are used, and the reports say so: read the total and the short segments as rough, the long segments are the signal. A population allele-frequency file (`--AF-file`, for example from gnomAD) would replace the one default value with real frequencies per site and give better segment edges; neither the script nor the module uses one today, and its effect on the total has not been measured. - The centromere and heterochromatin bands (above) are taken out of every count; the step and the reports do it, so there is nothing to subtract by hand -- Long runs concentrated on one chromosome may indicate uniparental disomy; the reports flag a chromosome whose segments of 5 Mb or more add up to at least 20 Mb and to more than half of the length of all such autosomal segments (above). Confirm with the parents' genotypes; a methylation test of a listed imprinted region tests for its imprinting disorder +- Long runs concentrated on one chromosome may indicate uniparental disomy; the reports flag a chromosome whose segments of 5 Mb or more add up to at least 20 Mb and to more than half of the length of all such autosomal segments (above). The flag requires sites on all 22 autosomes. Check copy number and acquired LOH, and confirm with the parents' genotypes. A methylation test assesses an imprinting defect - ROH within known disease gene regions warrants checking for homozygous pathogenic variants diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index 2724c416d..11a538c28 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -289,7 +289,7 @@ pub fn section(c: &Ctx<'_>) -> Out { let has_sites = assessed.iter().any(|yes| *yes); // Old -O r files omit ST rows. Their runs give a lower bound on // chromosome scope, never evidence for a genome-wide interpretation. - let covered = std::array::from_fn(|i| assessed[i] || observed[i]); + let covered: [bool; 22] = std::array::from_fn(|i| assessed[i] || observed[i]); let denominator = covered_unmasked_bp(&covered); let complete = has_sites && assessed.iter().all(|yes| *yes) diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 4f34ef10f..e4cad1c0e 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -259,6 +259,12 @@ fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1717)); assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(93_162_635)); + let mixed = roh_scope_summary(&[20, 22], "RG\tS\tchr20\t1000001\t9000000\t8000000\n"); + assert_eq!(at(&mixed, &["f_roh"]), &Value::Float(0.0859)); + assert_eq!( + at(&mixed, &["scope", "unmasked_bp"]), + &Value::Int(93_162_635) + ); let no_runs = roh_scope_summary(&[20], ""); assert_eq!(at(&no_runs, &["f_roh"]), &Value::Float(0.0)); assert_eq!(at(&no_runs, &["relationship"]), &Value::Null); diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f4dd8e3cd..d49f98629 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -41,7 +41,7 @@ 60-nextflow - 90-report-assets - acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh @@ -169,7 +169,7 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh diff --git a/tests/e2e/align-read-count-gate-1.sh b/tests/e2e/align-read-count-gate-1.sh index 0fe8e1dcb..c9edca8dc 100755 --- a/tests/e2e/align-read-count-gate-1.sh +++ b/tests/e2e/align-read-count-gate-1.sh @@ -50,7 +50,7 @@ TT="${T}/qc/${T}_read_count.tsv" check_eq "input_reads equals fastp's total_reads" "$(val input_reads "$TT")" "$FASTP" check_eq "the BAM's primary records equal fastp's total_reads" "$(primary "${T}/aligned/${T}_sorted.bam")" "$FASTP" check_eq "read_count_check" "$(val read_count_check "$TT")" ok -python3 "${REPO}/bin/collect_summary.py" --sample "$T" --sample-dir "${G}/${T}" --out "${CASE_TMP}/T.json" >/dev/null +"${OG:-${REPO}/bin/og}" summary collect --sample "$T" --sample-dir "${G}/${T}" --out "${CASE_TMP}/T.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/T.json" -o "${CASE_TMP}/T.txt" >/dev/null check "the text report shows both counts" \ has "^ Alignment: ${FASTP} reads in, ${FASTP} primary records in the BAM\$" "$(cat "${CASE_TMP}/T.txt" 2>/dev/null)" @@ -141,7 +141,7 @@ check_eq "the published table: primary_records" "$(val primary_records "${R}/qc/ check_eq "the published table: missing_reads" "$(val missing_reads "${R}/qc/${X}_read_count.tsv")" 8000 check_eq "the published BAM's primary records" "$(primary "${R}/aligned/${X}_sorted.bam")" 24000 check "the table is not published beside the BAM" test ! -e "${G}/${R}/aligned/${X}_read_count.tsv" -python3 "${REPO}/bin/collect_summary.py" --sample "$X" --sample-dir "${G}/${R}" --out "${CASE_TMP}/X.json" >/dev/null +"${OG:-${REPO}/bin/og}" summary collect --sample "$X" --sample-dir "${G}/${R}" --out "${CASE_TMP}/X.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.txt" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.html" >/dev/null check "the text report flags the missing reads" \ diff --git a/tests/e2e/vendor-vcf-intake-2-nf-report.sh b/tests/e2e/vendor-vcf-intake-2-nf-report.sh index 936730d4c..9fb6a49f6 100755 --- a/tests/e2e/vendor-vcf-intake-2-nf-report.sh +++ b/tests/e2e/vendor-vcf-intake-2-nf-report.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # A VCF-first Nextflow run (no BAM) with the tools a vendor-VCF user picks: # - the HTML report shows ROH, the mito haplogroup and CPIC, with the same -# ROH numbers and haplogroup bin/collect_summary.py (the bash report) reads +# ROH numbers and haplogroup og summary collect (the bash report) reads # from the same files, and it starts after the steps it shows; # - multiqc is selected but has nothing to read: the log says so in one line; # - the completion message prints, with no onComplete handler error; @@ -35,18 +35,18 @@ check "the report exists" test -s "$HTML" for h in 'Runs of Homozygosity' 'Mitochondrial Haplogroup' 'CPIC Drug Recommendations'; do check_eq "report card '${h}' is filled" "$(html_stat "$HTML" "$h" Status)" Complete done -python3 "${REPO}/bin/collect_summary.py" --sample "$SAMPLE" --sample-dir "$R" --out "${CASE_TMP}/summary.json" \ +"${OG:-${REPO}/bin/og}" summary collect --sample "$SAMPLE" --sample-dir "$R" --out "${CASE_TMP}/summary.json" \ > "${CASE_TMP}/summary.log" 2>&1 || cat "${CASE_TMP}/summary.log" want() { python3 -c "import json, sys; d = json.load(open('${CASE_TMP}/summary.json'))['sections']$1; print($2)" 2>/dev/null; } ROH_TOTAL=$(want "['roh']['data']['total_mb']" "'%.1f MB' % d") ROH_LARGEST=$(want "['roh']['data']['largest_mb']" "'%.1f MB' % d") HG=$(want "['haplogroup']['data']['haplogroup']" "d") -echo "collect_summary.py: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" -check "collect_summary.py read the ROH file" test -n "$ROH_TOTAL" +echo "og summary collect: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" +check "og summary collect read the ROH file" test -n "$ROH_TOTAL" check_eq "ROH total matches the bash report's" "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH total')" "$ROH_TOTAL" check_eq "ROH largest segment matches the bash report's" \ "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH largest segment')" "$ROH_LARGEST" -check "collect_summary.py read the haplogroup" test -n "$HG" +check "og summary collect read the haplogroup" test -n "$HG" check_eq "haplogroup matches the bash report's" "$(html_stat "$HTML" 'Mitochondrial Haplogroup' Haplogroup)" "$HG" # The report waits for the steps it shows. diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index 9c853a9e4..dd1380417 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -165,6 +165,11 @@ RG\tS\tchr1\t120000001\t145000000\t25000000\t500\t50 RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50 RG\tS\tchr15\t23000001\t50000000\t27000000\t900\t50 RG\tS\tchr3\t1\t2\tx\t1\t1\n" +# Sparse whole-genome scope, with chromosomes that have no RG rows too. +for chrom in {1..22}; do + printf 'ST\tS\tchr%s\t40000000\t0\t60\n' "$chrom" >> "$f/vcf/S_roh.txt" +done +at "$NEW" "$f/vcf/S_roh.txt" put $f/mito/S_haplogroup.txt "\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"H5a1\"\t\"0.9\"\n" put $f/mito/S_haplocheck.txt "\"Sample\"\t\"Contamination Status\"\t\"Contamination Level\"\n\"S\"\t\"NO\"\t\"\"\n" put $f/y_haplogroup/S_y_haplogroup.txt "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\nS\tR1b1a1b1a1a2*(xR-L21)\tR-M269\t100\t40\t0.98\n" @@ -248,7 +253,7 @@ put $f/kir/S_kir_genotype.tsv "# too few KIR reads\n" put $f/cnvpytor/S_cnvs.txt "deletion\tchrY:2800001-2900000\t100000\t0.0\t1e-15\t1\t1\t1\t0.05\t0\t9\ndeletion\tchr2:10000001-10100000\t100000\t0.4\t1e-9\n" at "$OLD" $f/cnvpytor/S_cnvs.txt put $f/telomere/x/deep/T_summary.tsv "PID\tsample\tintratelomeric_reads\ttel_content\nS\ttumor\t61873\t334\n" -put $f/roh/S_roh.txt "RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50\n" +put $f/roh/S_roh.txt "ST\tS\tchr2\t40000000\t0\t60\nRG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50\n" at "$OLD" $f/roh/S_roh.txt put $f/mito/S_haplogroup.txt "SampleID\tRank\nS\tU5a\n" at "$OLD" $f/mito/S_haplogroup.txt diff --git a/tests/parity/golden/summary-collect/all-bad/all-bad.json b/tests/parity/golden/summary-collect/all-bad/all-bad.json new file mode 100644 index 000000000..a20f67e6c --- /dev/null +++ b/tests/parity/golden/summary-collect/all-bad/all-bad.json @@ -0,0 +1,759 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: could not convert string to float: 'abc'", + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_callability.bed.gz is not mosdepth's thresholds table of one depth", + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "indexcov-indexcov.ped has no sex column or no sample row", + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sample_qc.tsv has no inferred_sex row", + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sample_qc.tsv has no mosaic_call row (detected, not_detected, gain, not_estimated)", + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_read_count.tsv has no whole number for input_reads", + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S.vcf.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: invalid literal for int() with base 10: 'x'", + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_couple_carrier.tsv line 2 is not a row of S's couple table", + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: invalid literal for int() with base 10: 'x'", + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S.report.json: Expecting value: line 1 column 21 (char 20)", + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [], + "non_normal": 0, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": true, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'Gene'", + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "none", + "filter": "" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_smn_copy_number.tsv lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or psCN_filter column", + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_paraphase_copy_number.tsv lacks the region, gene, copy_number or status column", + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01" + ], + "quality": [ + "x" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A low confidence" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancestry_fine.tsv has no rows or lacks a column of ['code', 'fraction', 'group', 'high', 'level', 'low', 'se']", + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sv_class4_5_checked.tsv lacks a column of ['ACMG_class', 'Caller', 'Common', 'Exon_edge', 'Population_AF', 'SV_chrom', 'SV_end', 'SV_start', 'SV_type']", + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 2, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "7/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "7-7/9-9", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/FLANKING", + "adsp": "8/0", + "ci": "11-11/12-12", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": false, + "unsized": [ + "CFTR_TG" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "not_sized", + "genotype": ".", + "phase": ".", + "reading": "ExpansionHunter did not size both tracts (TG 11/12, flanking_only; poly-T 7/9, sized): not read." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "not_sized", + "genotype": ".", + "pharmcat": ".", + "agreement": "not_compared", + "reading": "not in step 09's output: STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data adds the repeat to the catalog (docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 0, + "total_mb": 0.0, + "largest_mb": 0.0, + "autosomal_over_5mb": [], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": -0.0, + "f_roh": null, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "segments_only", + "autosomes": [], + "unmasked_bp": 0, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_haplocheck.txt has no Contamination Status column", + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_haplogroup.txt has no Hg column or no sample row", + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_tree.txt lacks a column of Hg, Level, Tree", + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_str.tsv has a confidence other than high, low, no call or not callable", + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_consumer_export.tsv has no whole-number 'sites' row", + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_bloodtype.tsv has no system, phenotype and confidence columns, or no row", + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_lpa_kiv2.tsv has no status row (estimated, too little depth or not assessed)", + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_vntyper.tsv has no status row (positive, possible, negative, not assessed)", + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_locityper.tsv lacks a row or one of the columns locus, region, status, genotype, haplotype_1, haplotype_2, quality, total_reads, unexplained_reads, weighted_distance, warnings, filter, reason", + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'length'", + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_traits.tsv line 2 does not have one field per column", + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_archaic.tsv has no percentage for neanderthal_percent", + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancient.tsv has no rows or lacks a column of ['country', 'dates', 'distance', 'group', 'locality', 'n', 'rank']", + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_viral.tsv has no rows or lacks a column of virus, accession, reads, breadth, relative_depth, copies_per_cell, call", + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_exomiser_top.tsv lacks a column of rank, gene, moi, exomiser_score, phenotype_score, variant, genotype, consequence, hgvs, acmg, disease", + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_chip.tsv lacks a column of gene, protein_change, vaf, alt_reads, depth, status, note", + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mei.tsv is missing beside S_mei_sites.tsv", + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mocha.tsv has 1 reported calls, S_mocha_summary.tsv says 2", + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S.cpsr.grch38.classification.tsv.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: invalid literal for int() with base 10: 'x'", + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_prioritized.vcf.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": "none" + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Polygenic risk scores", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all-history/all-history.json b/tests/parity/golden/summary-collect/all-history/all-history.json new file mode 100644 index 000000000..d3637eeb9 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-history/all-history.json @@ -0,0 +1,15569 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": "2021-03-01", + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [ + [ + "run", + "written_utc", + "2021-03-04 05:06:07 UTC" + ], + [ + "image", + "DEEPVARIANT_IMAGE", + "v1.10.0" + ], + [ + "data", + "clinvar_file_date", + "2021-03-01" + ] + ], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/indexcov-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "male", + "cn_chrX": "1.01", + "cn_chrY": "0.98", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "ok", + "source": "qc/S_read_count.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "input_reads": 1000, + "primary_records": 1000, + "missing_reads": 0, + "read_count_check": "ok" + } + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": 0.008, + "gnomad_genome_af": 0.007, + "rsids": [ + "rs113993960" + ], + "af": 0.008, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": 0.06, + "gnomad_genome_af": 0.05, + "rsids": [ + "rs99", + "rs1800562" + ], + "af": 0.06, + "common": true, + "cdna": { + "check": "clean", + "note": "", + "alt_reads": 20, + "alt_clipped": 0, + "ref_reads": 20, + "ref_clipped": 0, + "construct": null + } + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": 0.002, + "gnomad_genome_af": 0.0005, + "rsids": [ + "rs36053993" + ], + "af": 0.002, + "common": false, + "cdna": { + "check": "flagged", + "note": "possible cDNA/transgene contamination: check the reads", + "alt_reads": 12, + "alt_clipped": 6, + "ref_reads": 10, + "ref_clipped": 0, + "construct": "yes" + } + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": 0.03, + "gnomad_genome_af": null, + "rsids": [ + "rs80338939" + ], + "af": 0.03, + "common": true, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": "vep/S_vep.vcf.gz", + "note": null, + "common_af": 0.01 + }, + "cdna_flagged": 1, + "construct_sites": 1, + "cdna_scan": { + "note": "2 genes with spliced reads", + "loci": [ + { + "chrom": "chr1", + "start": 100, + "end": 200, + "junctions": 2, + "reads": 5, + "junction_list": "A:1-2,A:3-4" + } + ] + }, + "deep_intronic": [ + { + "chrom": "chr7", + "pos": 117639961, + "ref": "C", + "alt": "T", + "gene": "CFTR", + "hgvs": "c.3718-2477C>T", + "clinvar_id": "7129", + "call": "no call", + "detail": "" + } + ] + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy", + "hemizygous": "called homozygous inside one SMN1 copy (Parascopy, step 35)" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "SMN1", + "verdict": "biallelic", + "why": "the variant is called homozygous inside the deletion: it is hemizygous, on the one copy the deletion leaves", + "deletion": "one SMN1 copy (Parascopy, step 35)", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "allele": null, + "condition": "Spinal muscular atrophy", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)", + "Parascopy's SMN1 count (step 35)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not called" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "pharmcat/S.report.json", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "version": "3.4.0", + "html_report": true + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [ + "CYP2C19 has two possible phenotypes", + "last" + ] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + }, + "depth_check": { + "status": "ok", + "gene_mapq1_share": "0.99", + "body_mapq20_share": "0.95" + }, + "comparison": { + "conflicts": 1, + "one_tool_only": 2, + "concordant": 1 + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*2/*1", + "filter": "PASS" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": "male" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "-", + "paternal_upd": "-", + "pmid": "22237428;27615419" + } + ], + "note": "Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH." + }, + "scope": { + "source": "sites", + "autosomes": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22" + ], + "unmasked_bp": 2658001522, + "partial": false, + "note": "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *2/*1" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/S.cpsr.grch38.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "CLASSIFICATION", + "classification": { + "Pathogenic": 1, + "Likely_Pathogenic": 1, + "VUS": 1, + "Benign": 1, + ".": 1 + }, + "plp": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "13:32338000 G>A", + "gene": "BRCA2", + "final": "Likely_Pathogenic", + "cpsr": "VUS", + "clinvar": null, + "authority": "CPSR", + "acmg_codes": null, + "dp": 40, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "5:112839000 A>T", + "gene": "APC", + "final": ".", + "cpsr": "Pathogenic", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": 30, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "disagreements": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 1, + "call_floor_dp": 8, + "cdna_flagged": 1 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 5, + "by_impact": { + "HIGH": 3, + "MODERATE": 1, + ".": 1 + }, + "genes": 3, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + }, + "frequency_unknown": 1, + "weak_calls": 2 + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_slivar_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "frequency_unknown": 1, + "compound_het_variants": 4, + "compound_het_genes": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": "*1/*2", + "Cyrius": "*2/*1" + }, + "agree": true, + "consensus": { + "result": "*1/*2", + "passed_to_pharmcat": true, + "reason": "pypgx and Cyrius agree", + "source": "pgx_consensus/S_pgx_consensus.tsv" + }, + "probable": null + }, + "not_run": [], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ], + "recheck": { + "checked_utc": "2021-03-05 00:00:00 UTC", + "items": [ + { + "item": "clinvar", + "label": "ClinVar", + "class": "newer-release", + "ran_with": "2021-03-01", + "upstream": "2021-03-04", + "action": "rerun step 06" + }, + { + "item": "vep", + "label": "VEP", + "class": "no-newer-release", + "ran_with": null, + "upstream": null, + "action": null + } + ], + "affected_steps": [ + "06" + ], + "rerun": { + "command": "run-all.sh" + } + }, + "new_since": { + "previous_utc": "2021-01-01 00:00 UTC", + "sections": [ + "ancestry_fine", + "ancient", + "annotsv", + "archaic", + "bloodtype", + "callability", + "carrier", + "chip", + "clinical", + "cnvpytor", + "consumer_export", + "couple", + "coverage", + "cpsr", + "cyrius", + "delly", + "exomiser", + "expansions", + "haplogroup", + "hla", + "locityper", + "lpa", + "manta", + "mito", + "mosaic", + "mosaic_sex_loss", + "paralogs", + "parascopy", + "pharmcat", + "read_count", + "roh", + "sample_qc", + "sex_check", + "slivar", + "sv_consensus", + "telomere", + "traits", + "variants", + "viral", + "vntyper", + "y_haplogroup", + "y_str", + "y_tree", + "clinvar", + "cpic", + "pypgx", + "prs" + ] + } +} diff --git a/tests/parity/golden/summary-collect/all-nf/all-nf.json b/tests/parity/golden/summary-collect/all-nf/all-nf.json new file mode 100644 index 000000000..ec0c1438c --- /dev/null +++ b/tests/parity/golden/summary-collect/all-nf/all-nf.json @@ -0,0 +1,14043 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "female" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "coverage/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": NaN + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "coverage/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/A-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "female", + "cn_chrX": "2.1", + "cn_chrY": "0.02", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": null, + "note": "no VEP output (step 13) in this folder", + "common_af": 0.01 + } + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hom", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "CFTR", + "verdict": "possibly biallelic", + "why": "a heterozygous call inside a called deletion: one of the two calls may be wrong; check the reads", + "deletion": "CFTR_del deletion, 1 of 2 copies (read depth, step 35)", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "allele": null, + "condition": "Cystic fibrosis", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not assessed: S_hba_copy_number.tsv has no status row", + "Parascopy's SMN1 count: not resolved" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "vcf/B.report.json", + "file_date": "2021-03-04 05:07 UTC", + "note": null, + "data": { + "version": "", + "html_report": false + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "stale", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 27 was not run in the run of 2020-01-01 00:00 UTC", + "data": { + "genes": [ + { + "gene": "CYP2D6", + "diplotype": "*4/*4", + "phenotype": "Poor", + "status": "not called" + } + ], + "non_normal": 0, + "ambiguous": 0, + "not_called": 1, + "unclassified": 0, + "parse_failed": false, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/T_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*1/*2", + "filter": "LowQ" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "no row covers exon 7", + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": "." + }, + "smn2": { + "cn": null, + "text": "no row covers exon 7" + }, + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": ".", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "SMN1 g.27134T>G: not assessed, step 35 wrote no S_smn1_marker.tsv (a run from before this reader, or the check did not run)." + }, + "rows": 1, + "hba": { + "status": "not assessed", + "message": "not assessed: S_hba_copy_number.tsv has no status row" + }, + "cn_intervals": { + "rows": 3, + "by_status": { + "called": 3 + }, + "changes": [ + { + "locus": "CFTR_del", + "name": "CFTR region", + "chrom": "chr7", + "start": "117500000", + "end": "117600000", + "copies": "1", + "expected": "2", + "change": "deletion" + }, + { + "locus": "DUP1", + "name": "x", + "chrom": "chr1", + "start": "1", + "end": "100", + "copies": "3", + "expected": "2", + "change": "duplication" + }, + { + "locus": "GJB6", + "name": "x", + "chrom": "chr13", + "start": "x", + "end": "2", + "copies": "1", + "expected": "2", + "change": "deletion" + } + ] + }, + "cyp21a2": null, + "dosage": null, + "reading": "SMN1 and SMN2 not assessed: no model: no carrier reading", + "loci": [] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "x", + "status": "called", + "detail": "." + } + ], + "other_regions": 0, + "other_regions_called": 0, + "variants": {}, + "smn1": { + "paraphase": null, + "parascopy": { + "cn": null, + "text": "no row covers SMN1 exon 7" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*03:01", + "DQA1*01:01" + ], + "quality": [ + "0", + "30" + ], + "low_confidence": true, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*04:01", + "DRB1*07:01" + ], + "quality": [ + "40", + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 low confidence", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A not typed" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "present", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [], + "reason": "OptiType did not run: no class I reads" + }, + "kir": { + "typed": [], + "not_called": [], + "reason": "too few KIR reads" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "stale", + "source": "prs/S_prs_summary.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 25 was failed (exit 2) in the run of 2020-01-01 00:00 UTC", + "data": { + "scores": [ + { + "condition": "X", + "pgs_id": "PGS1", + "score": "", + "matched": "", + "total": "", + "chrx": "", + "ambiguous": "", + "applies_to": "", + "sex_check": "", + "percentile": "", + "group": "", + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": false, + "ancestry": { + "population": "", + "panel": "", + "low_confidence": "" + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/x/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 0, + "pass": 0 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "stale", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 18 was skipped (opt-in) in the run of 2020-01-01 00:00 UTC", + "data": { + "total": 2, + "deletions": 2, + "duplications": 0, + "significant": 2, + "significant_filtered": 1, + "filtered_chry": 1, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0, + "chry_female": 1 + }, + "declared_sex": "female" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "stale", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 22 was failed (exit 1) in the run of 2020-01-01 00:00 UTC", + "data": { + "consensus": 1 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 0, + "rows": [], + "downgraded": [], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 0, + "exon_edge": 0, + "exon_edge_checked": true, + "common": 0 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 3, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "5/5", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/5-5", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "12/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "12-12/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "UGT1A1", + "repeat_count": "8/10", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "8-8/10-10", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": true, + "unsized": [], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG12-5T/TG12-5T", + "pairings": [ + "TG12-5T/TG12-5T" + ], + "tg": [ + 12, + 12 + ], + "poly_t": [ + 5, + 5 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 2 copys with the 5T poly-T tract, TG12 on the 5T copies. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*28/TA9", + "ta_units": [ + 8, + 10 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*28/TA9 (TA7/TA9, 8/10 TA units as ExpansionHunter counts them): a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/x/deep/T_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "334", + "intratelomeric_reads": 61873, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "roh/S_roh.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": null, + "data": { + "segments": 1, + "total_mb": 7.0, + "largest_mb": 7.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + } + ], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": 7.0, + "f_roh": 0.0294, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "sites", + "autosomes": [ + "chr2" + ], + "unmasked_bp": 237993529, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "stale", + "source": "mito/S_haplogroup.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 12 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "haplogroup": "U5a", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/U5a" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/U5a/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "insufficient markers", + "valid_markers": ".", + "qc_score": ".", + "explore": [] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "no branch has a derived call", + "level": ".", + "tree": ".", + "path_derived": ".", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": ".", + "min_depth": ".", + "explore": [], + "path": [] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 10, + "called": 10, + "no_call": 0, + "from_reference_blocks": 0, + "template_sites": ".", + "file_23andme": ".", + "file_ancestrydna": ".", + "no_call_reasons": {} + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": ".", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "LPA KIV-2 4925G>A: not assessed, step 46 wrote no S_lpa_kiv2_4925.tsv (a run from before this reader, or the check did not run)." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": ".", + "status": "not_covered", + "risk_alleles": null + } + ], + "risk_alleles": 0, + "flag": ".", + "flag_text": "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "confidence": ".", + "variant": ".", + "flag": ".", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "." + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "stale", + "source": "locityper/S_locityper.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 53 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*04:01/DRB1*07:01" + }, + { + "locus": "HLA-DQB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DQB1*02:01" + }, + { + "locus": "CYP2D6", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *1/*2 (not usable: Filter LowQ)" + } + ], + "called": 3, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + } + ], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 1, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "mito/S_chrM_annotated.tsv is older than the chrM calls: rerun step 20" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [], + "called": 0, + "not_covered": 1, + "other_allele": 0, + "rules": null + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "not assessed (no human depth)", + "human_depth": ".", + "found": [], + "genomes": 1 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [], + "genes": [] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [], + "other": 0, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + } + ], + "median_depth": null, + "median_limit": null, + "not_covered": [ + "DNMT3A" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 0, + "by_element": {}, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "F", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "seen" + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "message": "too shallow", + "depth": ".", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": ".", + "calls_mocha": ".", + "calls_reported": ".", + "phasing": ".", + "calls": [], + "not_reported": 0 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/T.cpsr.grch37.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "FINAL_CLASSIFICATION", + "classification": { + "Pathogenic": 1, + ".": 1 + }, + "plp": [ + { + "variant": ".", + "gene": "A", + "final": "Pathogenic", + "cpsr": "Benign", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false + } + ], + "disagreements": [], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 0, + "call_floor_dp": 8 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 6, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + } + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_prioritized.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": "not called", + "pypgx": "*1/*2", + "Cyrius": "*1/*2 (not usable: Filter LowQ)" + }, + "agree": null, + "consensus": null, + "probable": { + "diplotype": "*1/*2", + "phenotype": "Normal Metabolizer", + "label": "probable (one caller: pypgx)" + } + }, + "not_run": [ + "Mosaic loss of chromosome Y or X", + "Variant calling", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Delly)", + "Y-STR markers (HipSTR)", + "Blood groups (predicted)", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all/all.json b/tests/parity/golden/summary-collect/all/all.json new file mode 100644 index 000000000..0ccd9b68a --- /dev/null +++ b/tests/parity/golden/summary-collect/all/all.json @@ -0,0 +1,15495 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": "2026-10-01", + "cpsr_clinvar_release": "20250901", + "hla_database": "IPD-IMGT/HLA 3.60\textra" + }, + "manifest": [ + [ + "run", + "declared_sex", + "male" + ], + [ + "data", + "clinvar_file_date", + "2026-10-01" + ], + [ + "data", + "cpsr_clinvar_release", + "20250901" + ], + [ + "data", + "hla_database", + "IPD-IMGT/HLA 3.60\textra" + ] + ], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/indexcov-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "male", + "cn_chrX": "1.01", + "cn_chrY": "0.98", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "ok", + "source": "qc/S_read_count.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "input_reads": 1000, + "primary_records": 1000, + "missing_reads": 0, + "read_count_check": "ok" + } + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": 0.008, + "gnomad_genome_af": 0.007, + "rsids": [ + "rs113993960" + ], + "af": 0.008, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": 0.06, + "gnomad_genome_af": 0.05, + "rsids": [ + "rs99", + "rs1800562" + ], + "af": 0.06, + "common": true, + "cdna": { + "check": "clean", + "note": "", + "alt_reads": 20, + "alt_clipped": 0, + "ref_reads": 20, + "ref_clipped": 0, + "construct": null + } + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": 0.002, + "gnomad_genome_af": 0.0005, + "rsids": [ + "rs36053993" + ], + "af": 0.002, + "common": false, + "cdna": { + "check": "flagged", + "note": "possible cDNA/transgene contamination: check the reads", + "alt_reads": 12, + "alt_clipped": 6, + "ref_reads": 10, + "ref_clipped": 0, + "construct": "yes" + } + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": 0.03, + "gnomad_genome_af": null, + "rsids": [ + "rs80338939" + ], + "af": 0.03, + "common": true, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": "vep/S_vep.vcf.gz", + "note": null, + "common_af": 0.01 + }, + "cdna_flagged": 1, + "construct_sites": 1, + "cdna_scan": { + "note": "2 genes with spliced reads", + "loci": [ + { + "chrom": "chr1", + "start": 100, + "end": 200, + "junctions": 2, + "reads": 5, + "junction_list": "A:1-2,A:3-4" + } + ] + }, + "deep_intronic": [ + { + "chrom": "chr7", + "pos": 117639961, + "ref": "C", + "alt": "T", + "gene": "CFTR", + "hgvs": "c.3718-2477C>T", + "clinvar_id": "7129", + "call": "no call", + "detail": "" + } + ] + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy", + "hemizygous": "called homozygous inside one SMN1 copy (Parascopy, step 35)" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "SMN1", + "verdict": "biallelic", + "why": "the variant is called homozygous inside the deletion: it is hemizygous, on the one copy the deletion leaves", + "deletion": "one SMN1 copy (Parascopy, step 35)", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "allele": null, + "condition": "Spinal muscular atrophy", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)", + "Parascopy's SMN1 count (step 35)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not called" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "pharmcat/S.report.json", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "version": "3.4.0", + "html_report": true + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [ + "CYP2C19 has two possible phenotypes", + "last" + ] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + }, + "depth_check": { + "status": "ok", + "gene_mapq1_share": "0.99", + "body_mapq20_share": "0.95" + }, + "comparison": { + "conflicts": 1, + "one_tool_only": 2, + "concordant": 1 + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*2/*1", + "filter": "PASS" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": "male" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "-", + "paternal_upd": "-", + "pmid": "22237428;27615419" + } + ], + "note": "Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH." + }, + "scope": { + "source": "sites", + "autosomes": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22" + ], + "unmasked_bp": 2658001522, + "partial": false, + "note": "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *2/*1" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/S.cpsr.grch38.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "CLASSIFICATION", + "classification": { + "Pathogenic": 1, + "Likely_Pathogenic": 1, + "VUS": 1, + "Benign": 1, + ".": 1 + }, + "plp": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "13:32338000 G>A", + "gene": "BRCA2", + "final": "Likely_Pathogenic", + "cpsr": "VUS", + "clinvar": null, + "authority": "CPSR", + "acmg_codes": null, + "dp": 40, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "5:112839000 A>T", + "gene": "APC", + "final": ".", + "cpsr": "Pathogenic", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": 30, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "disagreements": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 1, + "call_floor_dp": 8, + "cdna_flagged": 1 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 5, + "by_impact": { + "HIGH": 3, + "MODERATE": 1, + ".": 1 + }, + "genes": 3, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + }, + "frequency_unknown": 1, + "weak_calls": 2 + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_slivar_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "frequency_unknown": 1, + "compound_het_variants": 4, + "compound_het_genes": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": "*1/*2", + "Cyrius": "*2/*1" + }, + "agree": true, + "consensus": { + "result": "*1/*2", + "passed_to_pharmcat": true, + "reason": "pypgx and Cyrius agree", + "source": "pgx_consensus/S_pgx_consensus.tsv" + }, + "probable": null + }, + "not_run": [], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ] +} diff --git a/tests/parity/golden/summary-collect/c-bad1/bad1.json b/tests/parity/golden/summary-collect/c-bad1/bad1.json new file mode 100644 index 000000000..ad14386d1 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-bad1/bad1.json @@ -0,0 +1,762 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:18 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_couple_carrier.tsv line 2 is not a row of S's couple table", + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_smn_copy_number.tsv lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or psCN_filter column", + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_paraphase_copy_number.tsv lacks the region, gene, copy_number or status column", + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01" + ], + "quality": [ + "x" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A low confidence" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancestry_fine.tsv has no rows or lacks a column of ['code', 'fraction', 'group', 'high', 'level', 'low', 'se']", + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sv_class4_5_checked.tsv lacks a column of ['ACMG_class', 'Caller', 'Common', 'Exon_edge', 'Population_AF', 'SV_chrom', 'SV_end', 'SV_start', 'SV_type']", + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 2, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "7/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "7-7/9-9", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/FLANKING", + "adsp": "8/0", + "ci": "11-11/12-12", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": false, + "unsized": [ + "CFTR_TG" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "not_sized", + "genotype": ".", + "phase": ".", + "reading": "ExpansionHunter did not size both tracts (TG 11/12, flanking_only; poly-T 7/9, sized): not read." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "not_sized", + "genotype": ".", + "pharmcat": ".", + "agreement": "not_compared", + "reading": "not in step 09's output: STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data adds the repeat to the catalog (docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 0, + "total_mb": 0.0, + "largest_mb": 0.0, + "autosomal_over_5mb": [], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": -0.0, + "f_roh": null, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "segments_only", + "autosomes": [], + "unmasked_bp": 0, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_haplocheck.txt has no Contamination Status column", + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_haplogroup.txt has no Hg column or no sample row", + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_tree.txt lacks a column of Hg, Level, Tree", + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_str.tsv has a confidence other than high, low, no call or not callable", + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_consumer_export.tsv has no whole-number 'sites' row", + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_bloodtype.tsv has no system, phenotype and confidence columns, or no row", + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_lpa_kiv2.tsv has no status row (estimated, too little depth or not assessed)", + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_vntyper.tsv has no status row (positive, possible, negative, not assessed)", + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_locityper.tsv lacks a row or one of the columns locus, region, status, genotype, haplotype_1, haplotype_2, quality, total_reads, unexplained_reads, weighted_distance, warnings, filter, reason", + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'length'", + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_traits.tsv line 2 does not have one field per column", + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_archaic.tsv has no percentage for neanderthal_percent", + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancient.tsv has no rows or lacks a column of ['country', 'dates', 'distance', 'group', 'locality', 'n', 'rank']", + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_viral.tsv has no rows or lacks a column of virus, accession, reads, breadth, relative_depth, copies_per_cell, call", + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_exomiser_top.tsv lacks a column of rank, gene, moi, exomiser_score, phenotype_score, variant, genotype, consequence, hgvs, acmg, disease", + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_chip.tsv lacks a column of gene, protein_change, vaf, alt_reads, depth, status, note", + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mei.tsv is missing beside S_mei_sites.tsv", + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mocha.tsv has 1 reported calls, S_mocha_summary.tsv says 2", + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Coverage (mosdepth)", + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Sample identity and contamination (somalier, VerifyBamID2)", + "Mosaic loss of chromosome Y or X", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "Polygenic risk scores", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/c-full/full.json b/tests/parity/golden/summary-collect/c-full/full.json new file mode 100644 index 000000000..3e4f956fc --- /dev/null +++ b/tests/parity/golden/summary-collect/c-full/full.json @@ -0,0 +1,2402 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:18 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": null + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "-", + "paternal_upd": "-", + "pmid": "22237428;27615419" + } + ], + "note": "Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH." + }, + "scope": { + "source": "sites", + "autosomes": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22" + ], + "unmasked_bp": 2658001522, + "partial": false, + "note": "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Coverage (mosdepth)", + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Sample identity and contamination (somalier, VerifyBamID2)", + "Mosaic loss of chromosome Y or X", + "ClinVar screen", + "PharmCAT", + "pypgx", + "Cyrius (CYP2D6)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ] +} diff --git a/tests/parity/golden/summary-collect/c-nf/nf.json b/tests/parity/golden/summary-collect/c-nf/nf.json new file mode 100644 index 000000000..e836d9048 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-nf/nf.json @@ -0,0 +1,1200 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:18 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "female" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "no row covers exon 7", + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": "." + }, + "smn2": { + "cn": null, + "text": "no row covers exon 7" + }, + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": ".", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "SMN1 g.27134T>G: not assessed, step 35 wrote no S_smn1_marker.tsv (a run from before this reader, or the check did not run)." + }, + "rows": 1, + "hba": { + "status": "not assessed", + "message": "not assessed: S_hba_copy_number.tsv has no status row" + }, + "cn_intervals": null, + "cyp21a2": null, + "dosage": null, + "reading": "SMN1 and SMN2 not assessed: no model: no carrier reading", + "loci": [] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "x", + "status": "called", + "detail": "." + } + ], + "other_regions": 0, + "other_regions_called": 0, + "variants": {}, + "smn1": { + "paraphase": null, + "parascopy": { + "cn": null, + "text": "no row covers SMN1 exon 7" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*03:01", + "DQA1*01:01" + ], + "quality": [ + "0", + "30" + ], + "low_confidence": true, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*04:01", + "DRB1*07:01" + ], + "quality": [ + "40", + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 low confidence", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A not typed" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "present", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [], + "reason": "OptiType did not run: no class I reads" + }, + "kir": { + "typed": [], + "not_called": [], + "reason": "too few KIR reads" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "stale", + "source": "prs/S_prs_summary.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 25 was failed (exit 2) in the run of 2020-01-01 00:00 UTC", + "data": { + "scores": [ + { + "condition": "X", + "pgs_id": "PGS1", + "score": "", + "matched": "", + "total": "", + "chrx": "", + "ambiguous": "", + "applies_to": "", + "sex_check": "", + "percentile": "", + "group": "", + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": false, + "ancestry": { + "population": "", + "panel": "", + "low_confidence": "" + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/x/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 0, + "pass": 0 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "stale", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 18 was skipped (opt-in) in the run of 2020-01-01 00:00 UTC", + "data": { + "total": 2, + "deletions": 2, + "duplications": 0, + "significant": 2, + "significant_filtered": 1, + "filtered_chry": 1, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0, + "chry_female": 1 + }, + "declared_sex": "female" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "stale", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 22 was failed (exit 1) in the run of 2020-01-01 00:00 UTC", + "data": { + "consensus": 1 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 0, + "rows": [], + "downgraded": [], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 0, + "exon_edge": 0, + "exon_edge_checked": true, + "common": 0 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 3, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "5/5", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/5-5", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "12/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "12-12/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "UGT1A1", + "repeat_count": "8/10", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "8-8/10-10", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": true, + "unsized": [], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG12-5T/TG12-5T", + "pairings": [ + "TG12-5T/TG12-5T" + ], + "tg": [ + 12, + 12 + ], + "poly_t": [ + 5, + 5 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 2 copys with the 5T poly-T tract, TG12 on the 5T copies. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*28/TA9", + "ta_units": [ + 8, + 10 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*28/TA9 (TA7/TA9, 8/10 TA units as ExpansionHunter counts them): a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/x/deep/T_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "334", + "intratelomeric_reads": 61873, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "roh/S_roh.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": null, + "data": { + "segments": 1, + "total_mb": 7.0, + "largest_mb": 7.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + } + ], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": 7.0, + "f_roh": 0.0294, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "sites", + "autosomes": [ + "chr2" + ], + "unmasked_bp": 237993529, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "stale", + "source": "mito/S_haplogroup.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 12 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "haplogroup": "U5a", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/U5a" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/U5a/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "insufficient markers", + "valid_markers": ".", + "qc_score": ".", + "explore": [] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "no branch has a derived call", + "level": ".", + "tree": ".", + "path_derived": ".", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": ".", + "min_depth": ".", + "explore": [], + "path": [] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 10, + "called": 10, + "no_call": 0, + "from_reference_blocks": 0, + "template_sites": ".", + "file_23andme": ".", + "file_ancestrydna": ".", + "no_call_reasons": {} + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": ".", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "LPA KIV-2 4925G>A: not assessed, step 46 wrote no S_lpa_kiv2_4925.tsv (a run from before this reader, or the check did not run)." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": ".", + "status": "not_covered", + "risk_alleles": null + } + ], + "risk_alleles": 0, + "flag": ".", + "flag_text": "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "confidence": ".", + "variant": ".", + "flag": ".", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "." + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "stale", + "source": "locityper/S_locityper.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 53 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*04:01/DRB1*07:01" + }, + { + "locus": "HLA-DQB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DQB1*02:01" + }, + { + "locus": "CYP2D6", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + } + ], + "called": 3, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + } + ], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 1, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "mito/S_chrM_annotated.tsv is older than the chrM calls: rerun step 20" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [], + "called": 0, + "not_covered": 1, + "other_allele": 0, + "rules": null + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "not assessed (no human depth)", + "human_depth": ".", + "found": [], + "genomes": 1 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [], + "genes": [] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [], + "other": 0, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + } + ], + "median_depth": null, + "median_limit": null, + "not_covered": [ + "DNMT3A" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 0, + "by_element": {}, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "F", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "seen" + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "message": "too shallow", + "depth": ".", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": ".", + "calls_mocha": ".", + "calls_reported": ".", + "phasing": ".", + "calls": [], + "not_reported": 0 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Coverage (mosdepth)", + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Sample identity and contamination (somalier, VerifyBamID2)", + "Mosaic loss of chromosome Y or X", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Delly)", + "Y-STR markers (HipSTR)", + "Blood groups (predicted)", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 20eb59617..b2902119d 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -2113,7 +2113,7 @@ def report_sections_mito_roh_telomere_cnv(work, schema): def roh_upd_karyotype(work, schema): """F_ROH, the one-chromosome isodisomy flag joined to assets/imprinted_regions.tsv, and the sex-chromosome karyotype label.""" - # The catalog and the rows the summary holds (the Nextflow report task sees bin/, not assets/). + # The one table is embedded into og; the flag emits neutral region hints. with open(os.path.join(REPO, "assets/imprinted_regions.tsv")) as f: rows = list(csv.DictReader((l for l in f if not l.startswith("#")), delimiter="\t")) asset = [(r["chrom"], int(r["start"]) if r["start"] else None, int(r["end"]) if r["end"] else None, @@ -2126,55 +2126,55 @@ def roh_upd_karyotype(work, schema): # hg38.chrom.sizes) less the 217,000,000 bp of autosomal bands, written out # here rather than recomputed from the code under test. denom = 2875001522 - 217000000 + sites = "".join(f"ST\tS\tchr{c}\t40000000\t0\t60\n" for c in range(1, 23)) d = os.path.join(work, "roh_upd", "S") # Isodisomy: the whole of chr15 in one run (chr15 is 102 Mb; its first # 20.5 Mb are the acrocentric band, so 81.5 Mb count) and 8 Mb on chr2. upd = "RG\tS\tchr15\t1\t101991189\t101991189\t9000\t50\nRG\tS\tchr2\t10000001\t18000000\t8000000\t900\t50\n" - put(f"{d}/vcf/S_roh.txt", upd) + put(f"{d}/vcf/S_roh.txt", sites + upd) summ, txt, html = render(d) r = summ["sections"]["roh"]["data"] u = r.get("upd_flag") or {} long_bp = (101991189 - 20500000) + 8000000 check("ROH UPD: chr15 holding most of the long runs raises the flag", u.get("chrom") == "chr15" and u.get("mb") == 81.5 and r.get("relationship") is None, r) - check("ROH UPD: the flag joins the 15q11-q13 row, Prader-Willi from the mother and Angelman from the father", + check("ROH UPD: the flag names the 15q11-q13 region without a syndrome", [(x["region"], x["maternal_upd"], x["paternal_upd"]) for x in u.get("imprinted", [])] - == [("15q11-q13 (SNRPN, UBE3A)", "Prader-Willi syndrome", "Angelman syndrome")], u) + == [("15q11-q13 (SNRPN, UBE3A)", "-", "-")], u) check("ROH UPD: F_ROH of the two runs", abs((r.get("f_roh") or -1) - long_bp / denom) < 0.001, r.get("f_roh")) - check("ROH UPD: both reports carry the flag and the syndromes", - all("chr15 carries 81.5 MB" in t and "Prader-Willi syndrome" in t and "Angelman syndrome" in t + check("ROH UPD: both reports carry the flag and region without syndrome names", + all("chr15 carries 81.5 MB" in t and "15q11-q13 (SNRPN, UBE3A)" in t and "Prader-Willi" not in t and "Angelman" not in t for t in (txt, html)), txt) check("ROH UPD summary validates", not validate.validate(schema, json.loads(json.dumps(summ))), validate.validate(schema, json.loads(json.dumps(summ)))) - # Red control: the run on chr15 cut to 15q21 onwards misses 15q11-q13, so no row joins. - put(f"{d}/vcf/S_roh.txt", upd.replace("chr15\t1\t101991189\t101991189", "chr15\t40000001\t101991189\t61991189")) + # A distal isodisomic run can coexist with heterodisomy at the imprinted locus. + put(f"{d}/vcf/S_roh.txt", sites + upd.replace("chr15\t1\t101991189\t101991189", "chr15\t40000001\t101991189\t61991189")) u0 = section(d, "roh")["data"].get("upd_flag") or {} - check("ROH UPD red control: a chr15 run outside 15q11-q13 is flagged but joins no row", - u0.get("chrom") == "chr15" and u0.get("imprinted") == [], u0) + check("ROH UPD red control: a distal chr15 run still names the chromosome-wide region hint", + u0.get("chrom") == "chr15" and [x["region"] for x in u0.get("imprinted", [])] == ["15q11-q13 (SNRPN, UBE3A)"], u0) _, txt0, html0 = render(d) - check("ROH UPD: a flagged chr15 run outside 15q11-q13 says no listed region overlaps it, in both reports", - all("No listed imprinting-disorder region overlaps the flagged runs on chr15" in t - and "No imprinting disorder is listed" not in t for t in (txt0, html0)), txt0) + check("ROH UPD: a distal chr15 run lists its region without a syndrome in both reports", + all("15q11-q13 (SNRPN, UBE3A)" in t and "Prader-Willi" not in t and "Angelman" not in t for t in (txt0, html0)), txt0) check("ROH UPD: with a flag there is no band, so the band sentence of the F_ROH note is left out", all("F_ROH is the share of the autosomes" in t and "The band is the row" not in t for t in (txt0, html0)), txt0) # Just over half: 25.1 of 50 Mb is 50.2%, which a whole percent rounds to 50%. - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t35100000\t25100000\t900\t50\n" + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr2\t10000001\t35100000\t25100000\t900\t50\n" "RG\tS\tchr3\t10000001\t34900000\t24900000\t900\t50\n") _, txt5, html5 = render(d) check("ROH UPD: a share just over half reads more than 50%, never 50%, in both reports", all("chr2 carries 25.1 MB, more than 50% of the length" in t and "MB, 50%" not in t for t in (txt5, html5)), txt5) # chr7 joins as a whole chromosome. - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr7\t70000001\t110000000\t40000000\t900\t50\n") + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr7\t70000001\t110000000\t40000000\t900\t50\n") u7 = section(d, "roh")["data"].get("upd_flag") or {} - check("ROH UPD: 40 Mb on chr7 joins the whole-chromosome Silver-Russell row", - [x["maternal_upd"] for x in u7.get("imprinted", [])] == ["Silver-Russell syndrome"], u7) + check("ROH UPD: 40 Mb on chr7 lists a neutral region hint", + [(x["region"], x["maternal_upd"]) for x in u7.get("imprinted", [])] == [("7p12 (GRB10), 7q32 (MEST)", "-")], u7) # The same long total over 12 chromosomes: no flag, F_ROH the same, a relationship band. each = long_bp // 12 spread = "".join(f"RG\tS\tchr{c}\t10000001\t{10000000 + each}\t{each}\t900\t50\n" for c in range(1, 13)) - put(f"{d}/vcf/S_roh.txt", spread) + put(f"{d}/vcf/S_roh.txt", sites + spread) summ, txt, html = render(d) r = summ["sections"]["roh"]["data"] check("ROH spread: 12 chromosomes raise no flag", "upd_flag" in r and r["upd_flag"] is None, r) @@ -2187,12 +2187,12 @@ def roh_upd_karyotype(work, schema): and "may come from one parent" not in t for t in (txt, html)), txt) check("ROH spread: with a band the F_ROH note says what the band is", all("The band is the row" in t for t in (txt, html)), txt) # Short runs only: F_ROH 0, no sign of related parents. - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t13000000\t3000000\t900\t50\n") + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr2\t10000001\t13000000\t3000000\t900\t50\n") r = section(d, "roh")["data"] check("ROH: runs under 5 Mb only give F_ROH 0 and no sign of related parents", r.get("f_roh") == 0 and r.get("relationship") == "no sign of related parents", r) # A run of exactly 5 Mb counts, as in step 11's summary (5 Mb or more). - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t15000000\t5000000\t900\t50\n") + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr2\t10000001\t15000000\t5000000\t900\t50\n") r = section(d, "roh")["data"] check("ROH: a run of exactly 5 Mb counts in F_ROH and the long-run list, as in step 11", r.get("long_mb") == 5.0 and len(r.get("autosomal_over_5mb") or []) == 1, r) From c8bd5963ecf90ddcfa672aa94834ed5ce3f1dc53 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:23:43 +0100 Subject: [PATCH 56/86] docs(rust): name the four new crates, the planned verbs and the og-leg marker Bead: D1 Bead: D14 --- CHANGELOG.md | 3 +++ docs/rust.md | 8 +++++++- rust/README.md | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cac8762..9342e5038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. + - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/rust.md b/docs/rust.md index 0e0de2374..bfe8ead4b 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -31,6 +31,10 @@ rust/ crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers crates/og-recheck/ og recheck: newer databases since a run, the rerun and the ClinVar lock + crates/og-render/ og report render|index|charts: the reports from summary.json (not ported yet) + crates/og-live/ og live serve|results|family|demo, og view serve: the live site (not ported yet) + crates/og-couple/ og couple screen|genes: step 39's couple screen (not ported yet) + crates/og-admix/ what the fine-ancestry engines share: painting and projection (not ported yet) og/ the binary: og [args] bin/og the shim tasks call; runs bin/.og/og-linux- (og-darwin- on a Mac) tests/rust/ the differential test against CPython, the image test @@ -38,6 +42,8 @@ tests/rust/ the differential test against CPython, the image test Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report render|index|charts`, `og live serve|results|family|demo`, `og view serve`, `og couple screen|genes`, `og vcf index|shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og archaic compare|genomes`, `og ancestry-fine freqs|project|paint|simulate`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|params-doc|python-gate|everything-check|licences|docs-check|stub-http|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. + A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2`, `zlib-rs`, `memchr`, `smallvec`, `rustc-hash` and `rust-htslib` so far, declared in `rust/Cargo.toml` and locked in `Cargo.lock`). The musl builds also link `mimalloc` as their allocator: musl's own lets a long multi-threaded read hold several times its live memory. @@ -104,7 +110,7 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to ## CI -- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and the tests of the `og sv` and `og pgx` filters, the step 05 tests and `tests/test_ancestry_fine_report.py` run with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. +- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and every test that carries the line `# og-leg: OG_REQUIRED` (the tests that run `og`, such as the `og sv` and `og pgx` filter tests, the step 05 tests and `tests/test_ancestry_fine_report.py`) runs with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. A new test that needs `og` adds that line; the job has no list to edit. Before the real run the step plants a marked test with a failing assert, which must be picked and fail. The planned-verb test must fail once one verb's arm is dropped from `og/src/main.rs`. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. - **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a fresh and a stale macOS pair, a tree hash that cannot be computed, a release download with a bad checksum or attestation, without the macOS pair, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. - **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, runs the fetch smoke with both binaries (arm64 under QEMU) and with pysam in `PYPGX_IMAGE`, which must all match `fetch.pysam.tsv`, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. - **Release** (`release.yml`): a tag builds the four binaries, writes `SHA256SUMS`, attests them and attaches all five files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. diff --git a/rust/README.md b/rust/README.md index ce5fae19c..5a834a2e8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -22,6 +22,10 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-steps` | step helpers, among them `og pgx outside-calls`, `og pgx hla-concordance` (steps 08, 36), `og pgx pypgx-summary` (step 32) and `og paraphase check-bam`, `og paraphase table` (step 35b) | | `crates/og-fresh` | `og dev freshness`: the upstream lookups of the monthly freshness issue, each answer typed, HTTP through the host's `curl` | | `crates/og-recheck` | `og recheck`: which of a finished sample's results a newer ClinVar or catalog would change, `--rerun`, `--affected`, `--launch-dir` | +| `crates/og-render` | `og report render`, `og report index`, `og report charts`: the reports from summary.json (not ported yet) | +| `crates/og-live` | `og live serve`, `og live results`, `og live family`, `og live demo`, `og view serve`: the live site (not ported yet) | +| `crates/og-couple` | `og couple screen`, `og couple genes`: step 39's couple screen (not ported yet) | +| `crates/og-admix` | what the fine-ancestry engines share: our own painting and projection (not ported yet) | | `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | | `og.sh` | build or download `bin/.og/og-linux-{x86_64,aarch64}`, and check they match this `rust/` | | `rust-toolchain.toml` | the Rust release, the one `OG_BUILDER` (`versions.env`) ships | From a4fb1c573962ef11b20292e84a85aeb8695371b3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:24:16 +0100 Subject: [PATCH 57/86] fix(ci): keep ROH caller paths within the supported e2e scope Bead: lin-17 --- scripts/ci/e2e-case-paths.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index d49f98629..6dd7e7359 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -41,7 +41,7 @@ 60-nextflow - 90-report-assets - acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh @@ -169,7 +169,7 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh From c78f72afa1ba095f843211bc89df902bac803ec0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:24:30 +0100 Subject: [PATCH 58/86] fix(recheck): say the check failed when every lookup failed, and rerun with typed arguments A failed lookup no longer ends in 'Nothing here is newer than what the run used.': the last line says how many lookups failed, or 'Could not check' when none answered. The local comparison of what is installed here with what the run used runs before the lookup's error is read, so a newer ClinVar file on disk stays rerun-ready with its steps; the item keeps its error and the command still exits 1. --rerun reads 'meta command' back as the typed call run-all.sh records (cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS) and starts that run-all.sh with bash and its arguments one by one. Any other shape, or a cd folder that is gone, is refused with exit 2 and nothing runs. Bead: plat-25 Bead: plat-26 --- CHANGELOG.md | 2 ++ docs/recheck.md | 6 +++- rust/crates/og-recheck/src/cli.rs | 6 ++-- rust/crates/og-recheck/src/lib.rs | 53 +++++++++++++++++++++++------ rust/crates/og-recheck/src/rerun.rs | 24 ++++++++++++- scripts/run-all.sh | 9 ++--- 6 files changed, 81 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..574c7d7a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many lookups failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/recheck.md b/docs/recheck.md index c12066669..0d149ec73 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -49,6 +49,10 @@ Each item gets one class: | `unknown` | A pin of steps 50 to 53 that the run's manifest does not name (a run of an older version): nothing was compared | Nothing; the next run records it. It does not make the command exit 1 | | `error` | The lookup failed | Check again later. A failed lookup is never read as "up to date", and the command exits 1 | +A newer file or pin installed here stays `rerun-ready` when its lookup fails, because that comparison needs no network. The item then keeps its `error` too, and the command still exits 1. + +The last line of the output sums up the check. When something is `rerun-ready` or `refresh-available`, it names the steps a rerun would redo. When a lookup failed, it says how many of how many failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. "Nothing here is newer than what the run used." comes only when every lookup answered and nothing is newer. + The 2-day margin for ClinVar is there because NCBI dates the file a day or two after the release in its `##fileDate`. A newer PGS file date means the catalog wrote the harmonised file again; it does not prove the weights changed. ## Which steps a change redoes @@ -64,7 +68,7 @@ bin/og recheck --sample your_sample --rerun bin/og recheck --sample your_sample --rerun --refresh-clinvar # refresh ClinVar first ``` -`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` runs that command again with those switches only, so a `TOOLS` set in your shell today does not change the run. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. +`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` reads that command back word by word: the `cd` folder, the switch assignments, the `run-all.sh` path and its arguments. It starts that `run-all.sh` with `bash`, in that folder, with the arguments one by one and those switches only, so a `TOOLS` set in your shell today does not change the run. No shell reads the command. A row of any other shape, for example one edited to add `; rm ...`, is refused with exit 2 and nothing runs; a `cd` folder that no longer exists is refused the same way. Check such a row, then run the command by hand. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. It refuses while a run of the sample is going on (`run_status.tsv` has no `finished_utc`), and while another `--rerun` of the sample holds `recheck/rerun.lock`. A run that was killed without its exit trap leaves no `finished_utc` either: run `run-all.sh` by hand once, and the next check can rerun again. diff --git a/rust/crates/og-recheck/src/cli.rs b/rust/crates/og-recheck/src/cli.rs index f30f1d66f..b3e509870 100644 --- a/rust/crates/og-recheck/src/cli.rs +++ b/rust/crates/og-recheck/src/cli.rs @@ -6,9 +6,9 @@ //! og recheck --launch-dir DIR the last bare `nextflow run` there, with -resume //! ``` //! -//! Exit codes: 0, 1 when a lookup failed, 2 on a usage error or a sample -//! with no manifest, 3 when a lock is held or a run is going on, else the -//! replay's own exit code. +//! Exit codes: 0, 1 when a lookup failed, 2 on a usage error, a sample +//! with no manifest or a recorded command `--rerun` will not run, 3 when a +//! lock is held or a run is going on, else the replay's own exit code. use crate::answers::{self, Answers, Memo, Offline}; use crate::files::{last_rerun, log_check, write_check}; diff --git a/rust/crates/og-recheck/src/lib.rs b/rust/crates/og-recheck/src/lib.rs index 1c5d8e4fb..b966f3159 100644 --- a/rust/crates/og-recheck/src/lib.rs +++ b/rust/crates/og-recheck/src/lib.rs @@ -352,11 +352,11 @@ fn truthy(v: &Option) -> Option<&str> { } /// The class of a looked-up item; may set its error or the release that -/// pins a newer one. +/// pins a newer one. What is installed here is compared with what the run +/// used before the lookup's answer is read: that needs no network, so a +/// failed lookup leaves a newer file or pin here `rerun-ready`, with its +/// error kept. pub fn classify(item: &mut Item, release: &Result, clinvar: bool) -> Class { - if item.error.is_some() { - return Class::Error; - } let (ran, inst) = (truthy(&item.ran_with), truthy(&item.installed)); if let (Some(i), Some(r)) = (inst, ran) { if ver(i) > ver(r) { @@ -373,6 +373,9 @@ pub fn classify(item: &mut Item, release: &Result, clinvar: boo return Class::RerunReady; } } + if item.error.is_some() { + return Class::Error; + } // ClinVar's date is in the manifest whenever step 06 had a file, so a run // without one cannot be compared with the file here. A pin with no run // value (HLA data not installed then) compares this checkout's. @@ -572,10 +575,12 @@ impl Check { Value::Object(m) } + /// The items whose lookup failed or that could not be compared: the + /// `error` items, and a `rerun-ready` item whose lookup failed. pub fn errors(&self) -> usize { self.items .iter() - .filter(|i| i.class == Class::Error) + .filter(|i| i.class == Class::Error || i.error.is_some()) .count() } } @@ -747,7 +752,12 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( let or_unknown = |v: &Option| truthy(v).unwrap_or("unknown").to_owned(); for i in &c.items { let mut detail = match &i.error { - Some(e) if !e.is_empty() => e.clone(), + Some(e) if !e.is_empty() && i.class == Class::Error => e.clone(), + Some(e) if !e.is_empty() => format!( + "run used {}, here {}; the lookup failed: {e}", + or_unknown(&i.ran_with), + or_unknown(&i.installed), + ), _ => format!( "run used {}, here {}, upstream {}", or_unknown(&i.ran_with), @@ -774,15 +784,38 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( )?; } } - if c.affected_steps.is_empty() { - writeln!(out, "Nothing here is newer than what the run used.") - } else { + if !c.affected_steps.is_empty() { writeln!( out, "A rerun would redo steps {} and the text report.", c.affected_steps.join(", ") - ) + )?; + } + // A failed lookup is never read as "nothing newer". An item that was + // looked up has an answer or an error; the others were never asked. + let asked = c + .items + .iter() + .filter(|i| i.upstream.is_some() || i.error.is_some()) + .count(); + let failed = c.errors(); + if failed == 0 { + if c.affected_steps.is_empty() { + writeln!(out, "Nothing here is newer than what the run used.")?; + } + } else if failed >= asked { + writeln!( + out, + "Could not check: all {asked} lookups failed (each line above says why). Check again later." + )?; + } else { + writeln!( + out, + "{failed} of {asked} lookups failed: this check is incomplete, so it cannot say \ + nothing else is newer. Check again later." + )?; } + Ok(()) } #[cfg(test)] diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index 78c9f242a..af72f1474 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -355,6 +355,24 @@ pub fn rerun( )); }; let switches = meta.get("switches").cloned().unwrap_or_default(); + let Some(call) = parse_call(&cmd, &switches) else { + return Ok(Rerun::Refused( + 2, + "the command in logs/run_status.tsv is not a run-all.sh call as run-all.sh records it \ + (cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS): not running it. Check it, \ + then run it by hand." + .into(), + )); + }; + if !Path::new(&call.dir).is_dir() { + return Ok(Rerun::Refused( + 2, + format!( + "the run's directory {} is gone: run the command by hand from where it should run.", + call.dir + ), + )); + } let started = now_utc(); // After --refresh-clinvar the new file is installed, so it is rerun-ready. let predicted = state_.rerun_ready_steps.clone(); @@ -362,10 +380,14 @@ pub fn rerun( let _ = std::io::stdout().flush(); append_log(log, &[[&started, "rerun-start", "", "", &cmd]])?; let mut c = Command::new("bash"); - c.arg("-c").arg(&cmd); + c.arg(&call.script).args(&call.args).current_dir(&call.dir); for name in switches.split_whitespace() { c.env_remove(name); } + for (name, value) in &call.env { + c.env(name, value); + } + c.env("PWD", &call.dir); let rc = c.status().map(|s| s.code().unwrap_or(1)).unwrap_or(127); let after = files(sample_dir)?; let cmp = compare( diff --git a/scripts/run-all.sh b/scripts/run-all.sh index acec2fb02..8a7dc17cf 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -37,7 +37,7 @@ # failed (logs/run_status.tsv). Results are published by hard link when work/ and GENOME_DIR share a filesystem, # you passed no -w, -c or --publish_dir_mode, and you run as root or Linux's fs.protected_hardlinks is 0 (then a user # may link the tasks' root-owned files). -# logs/run_status.tsv keeps this command with its switches: bin/owngenome_recheck.py --rerun replays it when a newer +# logs/run_status.tsv keeps this command with its switches: bin/og recheck --rerun replays it when a newer # ClinVar or catalog changes a result (docs/recheck.md). set -euo pipefail case "${1:-}" in -h|--help) sed -n '2,/^set -euo/p' "$0" | sed -e '$d' -e 's/^# \{0,1\}//'; exit 0 ;; esac @@ -46,9 +46,10 @@ SAMPLE=${1:-} SEX=${2:-} [ -z "$SEX" ] || echo "ERROR: sex must be 'male' or 'female', got '${SEX}'."; } >&2; exit 2; } ((BASH_VERSINFO[0] * 100 + BASH_VERSINFO[1] >= 404)) || { echo "ERROR: run-all.sh needs bash 4.4 or later (this is ${BASH_VERSION}); on macOS: brew install bash" >&2; exit 2; } shift 2; SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) USER_THREADS=${THREADS:-} -# This command as given, for bin/owngenome_recheck.py --rerun (logs/run_status.tsv, 'meta command'): the switches +# This command as given, for bin/og recheck --rerun (logs/run_status.tsv, 'meta command'): the switches # below that are set, then the arguments, each shell-quoted, run from this directory so relative paths still hold. -# 'meta switches' names them all, so a replay drops one set now that the run did not have. +# 'meta switches' names them all, so a replay drops one set now that the run did not have. The replay reads this +# shape back word by word and starts run-all.sh with those arguments, never through a shell; it refuses any other. SWITCHES=(GENOME_DIR SKIP_VALIDATION THREADS SKIP_TRIM INTERVALS ALIGN_DIR TOOLS KIR HLA_QC HPO REF_FASTA EH_CATALOG MANTA_CALL_REGIONS PARASCOPY_POPULATION PARAPHASE_REGIONS ANCESTRY_PANEL ANCESTRY_FINE_DIR ARCHAIC_DIR ARCHAIC_POP PGSC_CALC_DIR ANCIENT_PANEL_DIR VIRAL_REF MEGANE_KMERS MOCHA_DATA GRIDSS IMPUTATION SOMATIC EXTRA_CALLERS BENCHMARK @@ -63,7 +64,7 @@ export GENOME_DIR=${GENOME_DIR:?Set GENOME_DIR to your data directory} validate_sample "$SAMPLE" G=$GENOME_DIR S="${GENOME_DIR}/${SAMPLE}" LOG_DIR="${GENOME_DIR}/${SAMPLE}/logs" T=${TOOLS:-} C=${EXTRA_CALLERS:-} on() { [[ "${!1:-}" =~ ^(true|1)$ ]]; } -# The ClinVar files are shared by every sample of GENOME_DIR. bin/owngenome_recheck.py --refresh-clinvar holds this +# The ClinVar files are shared by every sample of GENOME_DIR. bin/og recheck --refresh-clinvar holds this # lock while it replaces them; the run holds it shared on fd 7 until it ends, so a run waits for a refresh going on # and a refresh refuses while a run goes on. Without flock (macOS: brew install flock) nothing waits here. CVLOCK="${G}/clinvar/.refresh.lock" From c406f622f0205a70880a74419f511907d37c54a5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:26:40 +0100 Subject: [PATCH 59/86] fix(archive): reject empty Nextflow checksum reports Exercise the actual module script blocks on the synthetic reads and reject successful checksum commands that produce no report. Record the local case time. Bead: plat-62 --- modules/local/cram_archive/main.nf | 4 ++-- scripts/ci/e2e-case-times.tsv | 2 +- tests/e2e/qc-3-cram-checksum.sh | 33 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/modules/local/cram_archive/main.nf b/modules/local/cram_archive/main.nf index c9b96c270..069e98eaa 100644 --- a/modules/local/cram_archive/main.nf +++ b/modules/local/cram_archive/main.nf @@ -76,7 +76,7 @@ process CRAM_ARCHIVE { } checksum_rows ${bam} > bam.checksum checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${meta.id}_sorted.cram > ${meta.id}_sorted.cram.checksum - if ! diff bam.checksum ${meta.id}_sorted.cram.checksum >&2; then + if [ ! -s bam.checksum ] || [ ! -s ${meta.id}_sorted.cram.checksum ] || ! diff bam.checksum ${meta.id}_sorted.cram.checksum >&2; then echo "ERROR: the CRAM content of ${meta.id} differs from its BAM" >&2 exit 1 fi @@ -136,7 +136,7 @@ process CRAM_TO_BAM { } checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${cram} > cram.checksum checksum_rows ${meta.id}_from_cram.bam > bam.checksum - if ! diff cram.checksum bam.checksum >&2; then + if [ ! -s cram.checksum ] || [ ! -s bam.checksum ] || ! diff cram.checksum bam.checksum >&2; then echo "ERROR: the restored BAM content differs from ${cram}" >&2 exit 1 fi diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 066ae31fb..0206946d3 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -179,4 +179,4 @@ y-str-1 shard 42 - y-tree-current-1-placement shard 21 nextflow-hardening-2-nf rust-z2e-readers shard 1 - rust-c2b-traits shard 2 - -qc-3-cram-checksum shard 40 - +qc-3-cram-checksum shard 34 - diff --git a/tests/e2e/qc-3-cram-checksum.sh b/tests/e2e/qc-3-cram-checksum.sh index e1df0c207..acb4ce323 100755 --- a/tests/e2e/qc-3-cram-checksum.sh +++ b/tests/e2e/qc-3-cram-checksum.sh @@ -112,4 +112,37 @@ check "faithful restore writes BAM and index" test -s "${A}/${C}_sorted.bam" -a sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/original.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/original.checksum" sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/${C}_sorted.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/restored.checksum" check "round trip preserves read content and explicit tags" cmp -s "${CASE_TMP}/original.checksum" "${CASE_TMP}/restored.checksum" +# Run the module's actual script blocks on the same synthetic records. Only +# Nextflow interpolation and version metadata are omitted by this harness. +# Literal Nextflow placeholders are substituted by sed, not by this shell. +# shellcheck disable=SC2016 +module_script() { + awk -v process="$1" ' + $0 == "process " process " {" {found=1} + found && /^ script:/ {script=1; next} + script && /^ """/ {body=1; next} + body && /cat <<-END_VERSIONS/ {exit} + body {print} + ' "${REPO}/modules/local/cram_archive/main.nf" | + sed -e 's/${meta.id}/module/g' -e 's/${task.cpus}/1/g' \ + -e 's|${reference}|reference/GRCh38_no_alt_analysis_set.fasta|g' \ + -e 's|${bam}|checksum/aligned/original.bam|g' \ + -e 's|${cram}|module_sorted.cram|g' \ + -e 's/\\\\/\\/g' -e 's/\\\$/\$/g' +} +# A successful command with no report must never certify a conversion. The +# other samtools verbs still run for real, so flagstat and quickcheck pass. +# The function expands its arguments in the container shell. +# shellcheck disable=SC2016 +EMPTY_CHECKSUM='samtools() { if [ "$1" = checksum ]; then return 0; else command samtools "$@"; fi; }' +for process in CRAM_ARCHIVE CRAM_TO_BAM; do + body=$(module_script "$process") + rc=0 + in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "$body" || rc=$? + check_eq "${process}: faithful module script passes" "$rc" 0 + rc=0 + in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "${EMPTY_CHECKSUM} +${body}" || rc=$? + check "${process}: empty checksum reports fail" test "$rc" -ne 0 +done finish From d70b5252c181a3313ed5e17917509c1bdd85a78c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:26:52 +0100 Subject: [PATCH 60/86] fix(setup): install clinical reference data by default Bead: D11 --- CHANGELOG.md | 2 + docs/00-reference-setup.md | 51 ++++++----- docs/13-vep-annotation.md | 4 +- docs/17-cpsr.md | 25 +++--- docs/getting-started.md | 12 ++- docs/hardware-requirements.md | 6 +- docs/troubleshooting.md | 17 ++++ scripts/ci/fake-docker/lib.sh | 43 ++++----- scripts/lib/common.sh | 64 +++++++++++++- scripts/setup.sh | 96 ++++++++++++--------- scripts/validate-setup.sh | 13 ++- tests/fake-docker/base-setup.sh | 1 + tests/fake-docker/setup-clinical-default.sh | 11 +++ versions.env | 1 + 14 files changed, 229 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..f78b639b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin and warns without stopping the remaining installs. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index dfa88ad7d..b8f108743 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -4,6 +4,20 @@ One-time downloads required before running the pipeline. Each heading below give > **Estimated time:** 1-3 hours depending on internet speed. The two VEP caches (26 GB for step 13, 24 GB for step 17) are the largest downloads of a default run. +A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. +These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. +Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 14 GB, and VEP 115 needs 52 GB. +Each check adds a 5 GB margin. An item that does not fit is skipped with a warning. Setup continues. + +```bash +./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads +./scripts/setup.sh --clinical-data "$GENOME_DIR" # install or retry only these three +``` + +An interrupted transfer resumes. Existing caches and bundles are kept. +Without these data, `run-all.sh` skips the affected steps and records the reason. +A bare `nextflow run` still needs its tools and data parameters. Setup does not change that selection. + ## GRCh38 Reference Genome The foundation for everything. All tools need this. The pipeline uses NCBI's **GRCh38 no-ALT analysis set**: chr1-22, X, Y and M, the unplaced and unlocalized scaffolds and the EBV genome, 195 sequences, with UCSC names (`chr1`, `chrM`). `setup.sh` downloads it, checks it and stores it as `reference/GRCh38_no_alt_analysis_set.fasta`: @@ -143,7 +157,7 @@ Required for step 5. `setup.sh` downloads them (about 5 GB, about 20 GB unpacked ## VEP Cache (~26 GB) -Ensembl Variant Effect Predictor annotation database. Required for step 13 and the pipeline's VEP. Without it, a run skips VEP, vcfanno, the clinical filter and slivar. Install it before the first run: +Ensembl Variant Effect Predictor annotation database. Required for step 13 and the pipeline's VEP. Setup installs it by default. Without it, a run skips VEP, vcfanno, the clinical filter and slivar. To install only this cache: ```bash ./scripts/setup.sh --vep-cache ${GENOME_DIR} @@ -178,38 +192,21 @@ Required for step 17 (CPSR cancer predisposition screening). Includes ClinVar, g > **Important:** PCGR 2.3.2 bundles VEP 115, which requires a **release-115** cache — different from the release-116 cache used by step 13 above. See the next section for the VEP 115 download. The bundle date (`PCGR_DATA_BUNDLE`) and the cache release (`PCGR_VEP_CACHE_RELEASE`) are in `versions.env` and move with `PCGR_IMAGE`. -```bash -mkdir -p ${GENOME_DIR}/pcgr_data -cd ${GENOME_DIR}/pcgr_data - -# Download (~7 GB) -wget -c https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.20260620.grch38.tgz +Setup installs it by default. To add the clinical data to an existing setup: -# Extract and organize into version-stamped directory -tar xzf pcgr_ref_data.20260620.grch38.tgz -mkdir -p 20260620 && mv data/ 20260620/ -# Creates: ${GENOME_DIR}/pcgr_data/20260620/data/ - -# Optional: delete the tarball to save 7 GB -# rm pcgr_ref_data.20260620.grch38.tgz +```bash +./scripts/setup.sh --clinical-data "$GENOME_DIR" ``` +The installer checks the archive against `PCGR_DATA_SHA256` in `versions.env`, then extracts it into a temporary folder. +Only a complete tree is moved to `pcgr_data/20260620/data/`. The archive is deleted after installation. +The [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) names the bundle's third-party databases. Their own terms apply. Setup prints the resource link before downloading. + ## VEP 115 Cache for CPSR (~24 GB) PCGR 2.3.2 (step 17) bundles VEP 115 internally, which needs the **release-115** cache. This is separate from the release-116 cache used by step 13. Both coexist in the same `vep_cache/` directory under different subdirectories (`116_GRCh38/` and `115_GRCh38/`). A `113_GRCh38/` cache left from PCGR 2.2.5 is no longer read and can be deleted. -```bash -mkdir -p ${GENOME_DIR}/vep_cache/tmp -cd ${GENOME_DIR}/vep_cache/tmp - -# Download VEP 115 cache (~24 GB) -wget -c https://ftp.ensembl.org/pub/release-115/variation/indexed_vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz - -# Extract alongside the existing release-116 cache -cd ${GENOME_DIR}/vep_cache -tar xzf tmp/homo_sapiens_vep_115_GRCh38.tar.gz -# Creates: ${GENOME_DIR}/vep_cache/homo_sapiens/115_GRCh38/ -``` +Setup installs this cache by default, after the PCGR bundle. `--clinical-data` retries it using Ensembl's `CHECKSUMS` file and a temporary extraction folder. > If you only run step 13 (VEP annotation) and skip step 17 (CPSR), you only need the release-116 cache. If you only run step 17, you only need release-115. @@ -510,7 +507,7 @@ wget -c https://giab.s3.amazonaws.com/release/AshkenazimTrio/HG002_NA24385_son/N [Hardware and storage requirements](hardware-requirements.md#shared-reference-data-one-time) has the one table of every download above, with the totals. -> **Tip:** If disk space is tight, you can skip the VEP cache (step 13) and PCGR bundle (step 17) initially. The core pipeline (steps 2-3-6-7) only needs the reference FASTA and ClinVar (~3.5 GB unpacked). +> **Tip:** If disk space is tight, use `setup.sh --no-clinical-data` to skip both VEP caches and the PCGR bundle initially. The core pipeline (steps 2-3-6-7) only needs the reference FASTA and ClinVar (~3.5 GB unpacked). ## Verifying Your Setup diff --git a/docs/13-vep-annotation.md b/docs/13-vep-annotation.md index c1e859f6e..b3c0ca43d 100644 --- a/docs/13-vep-annotation.md +++ b/docs/13-vep-annotation.md @@ -16,7 +16,7 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. ## Prerequisites - The VCF from step 3 **and its `.tbi`**: the step refuses a VCF without its index, which may be half written -- Offline VEP cache: step 13 downloads, checks and unpacks it the first time it runs (see step 00-reference-setup) +- Offline VEP cache: setup installs it by default. To retry, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"` ([reference setup](00-reference-setup.md#vep-cache-26-gb)). The pipeline skips VEP when the cache is missing. - Cache size: see [Hardware and storage requirements](hardware-requirements.md#shared-reference-data-one-time) (about 26 GB to download, 30 GB unpacked) ## Command @@ -112,4 +112,4 @@ After annotation, use step 23 (clinical filter) which automatically detects avai - `--dir_cache /opt/vep/.vep`: the cache is mounted there, not in the home directory VEP looks in by default - `--fasta`: running `--offline` without a FASTA file disables HGVS notation (`INFO: Disabling --hgvs`), so the script always passes the reference ([`REF_FASTA`](00-reference-setup.md#the-reference-path-on-every-page) is the reference path) - VEP does NOT assess variant pathogenicity in ClinVar context — combine with step 6 (ClinVar screen) for full picture -- **Upgrading from an older release:** VEP reads the cache directory named after the release passed in `--cache_version` (`homo_sapiens/116_GRCh38/` for the pinned release 116). When that directory is missing, VEP stops with an error instead of annotating from an older cache such as `112_GRCh38/`; step 13 downloads the matching cache on its first run. +- **Upgrading from an older release:** VEP reads the cache directory named after the release passed in `--cache_version` (`homo_sapiens/116_GRCh38/` for the pinned release 116). When that directory is missing, VEP stops with an error instead of annotating from an older cache such as `112_GRCh38/`. Setup installs the matching cache by default. The standalone step script can download a missing cache, but `run-all.sh` skips VEP until the cache is installed. diff --git a/docs/17-cpsr.md b/docs/17-cpsr.md index e542e1feb..e0284dda8 100644 --- a/docs/17-cpsr.md +++ b/docs/17-cpsr.md @@ -22,22 +22,21 @@ The image is PCGR 2.3.2 from `ghcr.io/sigven/pcgr` (Docker Hub's `sigven/pcgr` s ### 1. VEP Cache PCGR 2.3.2 bundles VEP 115, which requires the **release-115** cache (~24 GB). This is different from the release-116 cache used by step 13. Both coexist in the same `vep_cache/` directory under different subdirectories (`116_GRCh38/` and `115_GRCh38/`). -```bash -mkdir -p ${GENOME_DIR}/vep_cache -wget -c -P ${GENOME_DIR}/vep_cache https://ftp.ensembl.org/pub/release-115/variation/indexed_vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz -tar xzf ${GENOME_DIR}/vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz -C ${GENOME_DIR}/vep_cache -``` -### 2. PCGR Ref Data Bundle -PCGR 2.x uses a ref data bundle (~7 GB for `20260620`) separate from VEP: +### 2. PCGR ref data bundle +PCGR 2.x uses a separate ref data bundle (`20260620`, about 7 GB). +A plain setup installs the VEP 115 cache and this bundle by default, alongside VEP 116 for step 13. +For an existing setup, install or retry all three with: + ```bash -mkdir -p ${GENOME_DIR}/pcgr_data -cd ${GENOME_DIR}/pcgr_data -wget -c https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.20260620.grch38.tgz -tar xzf pcgr_ref_data.20260620.grch38.tgz -mkdir -p 20260620 && mv data/ 20260620/ +./scripts/setup.sh --clinical-data "$GENOME_DIR" ``` -This creates a `20260620/data/` directory with ClinVar, CancerMine, UniProt, and other databases. VEP cache is now mounted separately. + +The bundle goes to `pcgr_data/20260620/data/`; the caches coexist under `vep_cache/homo_sapiens/`. +Setup checks Ensembl's cache checksums and the bundle's pinned SHA-256 before extraction, then removes each archive. +It checks free space for each install and continues with a warning if one fails or does not fit. +`--no-clinical-data` skips all three. Without either CPSR item, `run-all.sh` skips step 17. +See [reference setup](00-reference-setup.md) for disk peaks and the [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) for the bundled databases. Review each source's terms before use. ### Upgrading from PCGR 2.2.5 PCGR 2.2.5 read bundle `20250314` and the release-113 cache. 2.3.2 needs both new ones: about 31 GB to download (the 7 GB bundle and the 24 GB cache). Once step 17 runs on 2.3.2, `pcgr_data/20250314/` and `vep_cache/homo_sapiens/113_GRCh38/` are no longer read and can be deleted. diff --git a/docs/getting-started.md b/docs/getting-started.md index 6067bc415..e72239787 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,7 +18,7 @@ | **CPU** | 4 cores | 8-16 cores | `run-all.sh` runs DeepVariant with at most 8 shards, fewer when `--max_cpus` is lower; more cores run other steps beside it ([CPU](hardware-requirements.md#cpu-requirements)) | | **RAM** | 16 GB | 32 GB | Every container has a hard memory limit; the 8-CPU steps ask for 32 GB, capped at the machine's RAM | | **Disk** | 500 GB free from a BAM, about 700 GB from FASTQ | 1 TB+ | See [detailed breakdown](hardware-requirements.md#disk-space-breakdown) | -| **Internet** | Broadband | 100+ Mbps | ~70-75 GB core downloads + ~175 GB optional annotation databases | +| **Internet** | Broadband | 100+ Mbps | ~73-78 GB core downloads + ~175 GB optional annotation databases | | **OS** | Linux (amd64) | Ubuntu 22.04+ | macOS/ARM works but slower (see below) | > **Disk space is the #1 surprise.** A single 30X WGS sample produces 60-90 GB of FASTQ, 80-120 GB of BAM, plus reference genomes and databases. See [docs/hardware-requirements.md](hardware-requirements.md) for the full breakdown. @@ -46,11 +46,17 @@ Every analysis tool runs inside Docker: no conda environments, no Python version | PCGR/CPSR data bundle + VEP 115 cache | ~31 GB | Step 17 (cancer predisposition) | | Docker images (all steps) | ~10-15 GB | All steps | | Annotation databases (CADD, SpliceAI, REVEL, AlphaMissense) | ~175 GB | Steps 30-31 (optional) | -| **Total one-time setup (core)** | **~70-75 GB** | | +| **Total one-time setup (core)** | **~73-78 GB** | | | **Total with annotation enrichment** | **~250 GB** | | See [docs/00-reference-setup.md](00-reference-setup.md) for download instructions. +Plain setup includes VEP 116, the PCGR/CPSR bundle and VEP 115: about 57 GB of clinical downloads and 65 GB unpacked. +Allow 1 to 3 hours. Setup checks each install's disk peak plus a 5 GB margin and warns if it skips one. +Use `./scripts/setup.sh --no-clinical-data "$GENOME_DIR"` to opt out. +Use `./scripts/setup.sh --clinical-data "$GENOME_DIR"` to add or retry only these data later. +`run-all.sh` skips steps whose data are still missing; inspect its plan and setup warnings. + ## Platform Notes ### Linux (Recommended) @@ -120,7 +126,7 @@ Then, once per machine and once per sample: ```bash export GENOME_DIR=/path/to/your/data -./scripts/setup.sh $GENOME_DIR # reference, ClinVar, small data files, images and og (once) +./scripts/setup.sh $GENOME_DIR # reference, ClinVar, VEP caches, CPSR bundle, images and og (once) ./scripts/validate-setup.sh your_name # what is installed and what is missing ./scripts/run-all.sh your_name male # or female: sets the chrX/chrY ploidy and the sex check ``` diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index ba4ef1c61..0077edce2 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -43,7 +43,7 @@ A run from FASTQ needs more space than a run from a BAM. Until you delete ` "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38/info.txt" - tar -czf "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_${rel}_GRCh38.tar.gz" -C "${CASE_WORK}/cache" "homo_sapiens/${rel}_GRCh38" -done -for rel in 116 115; do - name="homo_sapiens_vep_${rel}_GRCh38.tar.gz" - printf '%s %s\n' "$(sum -r "${FAKE_DOWNLOAD_DIR}/${name}" | awk '{print $1, $2}')" "$name" -done > "${FAKE_DOWNLOAD_DIR}/CHECKSUMS" -mkdir -p "${CASE_WORK}/bundle/data/grch38" -printf 'fake PCGR data\n' > "${CASE_WORK}/bundle/data/grch38/README" -tar -czf "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/bundle" data -if command -v sha256sum >/dev/null 2>&1; then - PCGR_DATA_SHA256=$(sha256sum "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') -else - PCGR_DATA_SHA256=$(shasum -a 256 "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') -fi -export PCGR_DATA_SHA256 +seed_clinical_downloads() { + local rel name + export FAKE_DOWNLOAD_DIR="${CASE_WORK}/downloads" + mkdir -p "$FAKE_DOWNLOAD_DIR" + for rel in 116 115; do + mkdir -p "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38" + printf 'fake VEP cache\n' > "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38/info.txt" + tar -czf "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_${rel}_GRCh38.tar.gz" -C "${CASE_WORK}/cache" "homo_sapiens/${rel}_GRCh38" + done + for rel in 116 115; do + name="homo_sapiens_vep_${rel}_GRCh38.tar.gz" + printf '%s %s\n' "$(sum -r "${FAKE_DOWNLOAD_DIR}/${name}" | awk '{print $1, $2}')" "$name" + done > "${FAKE_DOWNLOAD_DIR}/CHECKSUMS" + mkdir -p "${CASE_WORK}/bundle/data/grch38" + printf 'fake PCGR data\n' > "${CASE_WORK}/bundle/data/grch38/README" + tar -czf "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/bundle" data + if command -v sha256sum >/dev/null 2>&1; then + PCGR_DATA_SHA256=$(sha256sum "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') + else + PCGR_DATA_SHA256=$(shasum -a 256 "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') + fi + export PCGR_DATA_SHA256 +} fail() { echo "ASSERT FAIL: $*" >&2 diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 74f3cfa2e..2088b40a6 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -632,6 +632,10 @@ install_vep_cache() { url=$(vep_cache_url "$rel") tarball="${dir}/$(basename "$url")" final="${dir}/homo_sapiens/${rel}_GRCh38" + if [ -f "${final}/info.txt" ]; then + echo "[OK] VEP ${rel} cache already present." + return 0 + fi if [ -e "$final" ]; then echo "ERROR: ${final} exists but has no info.txt (an interrupted extraction?)." >&2 echo " Remove it and run again." >&2 @@ -646,10 +650,68 @@ install_vep_cache() { echo "ERROR: could not unpack ${tarball} into a ${rel}_GRCh38 cache." >&2 return 1 fi - mv "${tmp}/homo_sapiens/${rel}_GRCh38" "$final" + if ! mv "${tmp}/homo_sapiens/${rel}_GRCh38" "$final"; then + rm -rf "$tmp" + return 1 + fi rm -rf "$tmp" "$tarball" } +# clinical_disk_ok DIR PEAK_GB LABEL: GB as du counts them (2^30 bytes), +# plus a 5 GB margin. +# Use the destination's filesystem, creating the directory when it is new. +# An unreadable df is a failed preflight, never permission to fill the disk. +clinical_disk_ok() { + local dir=$1 peak=$2 label=$3 free + mkdir -p "$dir" || return 1 + free=$(df -Pk "$dir" 2>/dev/null | awk 'NR == 2 {print $4}') || free="" + case "$free" in + ''|*[!0-9]*) echo "[WARN] Cannot read free space for ${label}; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; return 1 ;; + esac + if [ "$free" -lt "$(( (peak + 5) * 1048576 ))" ]; then + echo "[WARN] Skipping ${label}: needs ${peak} GB peak plus a 5 GB margin; $(awk -v k="$free" 'BEGIN {printf "%.1f", k / 1048576}') GB free." + echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" + return 1 + fi +} + +# install_pcgr_bundle DIR: check the pinned archive before extracting into a +# staging tree. Upstream ships data/ at the root; CPSR reads BUNDLE/data/. +# The shared fetch helper resumes interrupted transfers and deletes bad ones. +# shellcheck disable=SC2154 # PCGR_DATA_BUNDLE and PCGR_DATA_SHA256 are pins in versions.env +install_pcgr_bundle() { + local dir=$1 url tarball tmp + if [ -d "${dir}/${PCGR_DATA_BUNDLE}/data" ]; then + echo "[OK] PCGR/CPSR ${PCGR_DATA_BUNDLE} bundle already present." + return 0 + fi + echo "PCGR/CPSR reference data includes third-party databases with their own terms:" + echo " https://sigven.github.io/pcgr/articles/annotation_resources.html" + echo " Review those terms before use; --no-clinical-data skips this download." + url="https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" + tarball="${dir}/${url##*/}" + mkdir -p "$dir" || return 1 + fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 + tmp=$(mktemp -d "${dir}/.extract.XXXXXX") || return 1 + if ! tar xzf "$tarball" -C "$tmp" || [ ! -d "${tmp}/data/grch38" ]; then + rm -rf "$tmp" + echo "ERROR: could not unpack ${tarball} into PCGR data/grch38." >&2 + return 1 + fi + # An incomplete destination is never mistaken for a completed bundle. + if [ -e "${dir}/${PCGR_DATA_BUNDLE}" ]; then + rm -rf "$tmp" + echo "ERROR: ${dir}/${PCGR_DATA_BUNDLE} exists without data; remove it and retry." >&2 + return 1 + fi + if ! mv "$tmp" "${dir}/${PCGR_DATA_BUNDLE}"; then + rm -rf "$tmp" + return 1 + fi + rm -f "$tarball" + echo "[OK] PCGR/CPSR ${PCGR_DATA_BUNDLE} bundle installed." +} + # lock_acquire DIR / lock_release DIR: a lock shared by concurrent runs. The # lock is a directory holding the owner's PID. It is stale when that PID is # gone (a killed run), or when it has no PID a minute after it was made (a run diff --git a/scripts/setup.sh b/scripts/setup.sh index 326dfeddd..272808b4f 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # setup.sh — One-stop setup: download references, pull Docker images, validate # Usage: ./scripts/setup.sh +# ./scripts/setup.sh --no-clinical-data +# ./scripts/setup.sh --clinical-data # ./scripts/setup.sh --pull-only pull the Docker images and exit # ./scripts/setup.sh --og put OwnGenome's og binaries in bin/.og # (Linux, and macOS for a live-site host): @@ -82,8 +84,8 @@ # checkout of pgsc_calc and the Nextflow plugin it needs, so step 25 and # the PRS process run without the network # -# The VEP cache (~26 GB, --vep-cache) and PCGR ref data (~7 GB) are -# downloaded separately because they take a long time. +# 7. VEP 116, PCGR/CPSR data and VEP 115 (~57 GB download, ~65 GB unpacked). +# --no-clinical-data skips them; --clinical-data installs only these three. # # Every download goes through fetch (scripts/lib/common.sh): it is written to # .part, checked, and only then renamed, so a file that exists is whole. @@ -99,6 +101,9 @@ # existing samples. set -euo pipefail +# Keep an explicit checksum override for fixture tests and verified mirrors. +PCGR_SHA_OVERRIDE=${PCGR_DATA_SHA256:-} + # The usage text below names data releases, so read them before it prints. # shellcheck source=../versions.env . "$(dirname "$0")/../versions.env" @@ -107,12 +112,19 @@ PULL_ONLY=false REFRESH="" SAMPLE_QC_ONLY=false OPT_IN="" +CLINICAL_DATA=true case "${1:-}" in --og) exec "$(dirname "$0")/../rust/og.sh" install ;; --cyrius|--parascopy-data|--kir-data|--ancestry-panel|--ancestry-fine-panel|--yleaf-data|--consumer-sites|--ancient-panel|--viral-data|--masked-reference|--exomiser-data|--chip-data|--strchive-data|--megane-kmers|--mocha-data|--locityper-data|--vep-cache|--annotsv-data|--archaic) OPT_IN=${1#--} shift ;; + --clinical-data) + OPT_IN=clinical-data + shift ;; + --no-clinical-data) + CLINICAL_DATA=false + shift ;; --pull-only) PULL_ONLY=true shift ;; @@ -132,6 +144,8 @@ esac GENOME_DIR=${1:-${GENOME_DIR:-""}} if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo "Usage: $0 " + echo " $0 --no-clinical-data " + echo " $0 --clinical-data " echo " $0 --pull-only" echo " $0 --og" echo " $0 --refresh clinvar " @@ -140,6 +154,8 @@ if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo "" echo " Where to store reference data and sample outputs." echo " Needs at least 500 GB free space per sample." + echo " --no-clinical-data Skip both VEP caches and the PCGR/CPSR bundle." + echo " --clinical-data Install only those three downloads for an existing setup." echo " --pull-only Pull the Docker images listed in versions.env and exit." echo " --og Put OwnGenome's og binaries in bin/.og, for Linux and for a macOS live-site host" echo " (built from rust/, or a release's) and exit." @@ -210,6 +226,7 @@ export GENOME_DIR # Image tags, data versions and the helpers (run_in, fetch, pipeline_images) # shellcheck source=lib/common.sh . "$(dirname "$0")/lib/common.sh" +if [ -n "$PCGR_SHA_OVERRIDE" ]; then PCGR_DATA_SHA256=$PCGR_SHA_OVERRIDE; fi # NCBI's GRCh38 analysis set without ALT contigs: chr1-22, X, Y, M, the # unplaced and unlocalized scaffolds and chrEBV, 195 sequences. NCBI lists the @@ -2065,6 +2082,7 @@ install_vep_cache_opt_in() { echo "[OK] VEP ${VEP_CACHE_RELEASE} cache already present: ${dir}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38" return 0 fi + clinical_disk_ok "$dir" 56 "VEP ${VEP_CACHE_RELEASE} cache" || return 1 echo "Installing the VEP ${VEP_CACHE_RELEASE} cache: ~26 GB download, ~30 GB unpacked, both on disk until it ends." if ! install_vep_cache "$dir" "$VEP_CACHE_RELEASE"; then echo "[WARN] Could not install the VEP ${VEP_CACHE_RELEASE} cache. Run this command again: the download resumes." @@ -2073,6 +2091,33 @@ install_vep_cache_opt_in() { echo "[OK] VEP ${VEP_CACHE_RELEASE} cache: ${dir}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38" } +# Each missing item gets its own disk check. A skipped or failed item must +# not prevent the smaller bundle or the other cache from installing. +install_clinical_data() { + local dir="${GENOME_DIR}/vep_cache" rel peak + echo "Clinical data: about 57 GB to download, 65 GB unpacked; allow 1 to 3 hours." + for rel in "$VEP_CACHE_RELEASE" bundle "$PCGR_VEP_CACHE_RELEASE"; do + if [ "$rel" = bundle ]; then + [ ! -d "${GENOME_DIR}/pcgr_data/${PCGR_DATA_BUNDLE}/data" ] || { echo "[OK] PCGR/CPSR bundle already present."; continue; } + if clinical_disk_ok "${GENOME_DIR}/pcgr_data" 14 'PCGR/CPSR bundle'; then + install_pcgr_bundle "${GENOME_DIR}/pcgr_data" || echo "[WARN] Could not install PCGR/CPSR. Step 17 needs it; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" + fi + else + [ ! -f "${dir}/homo_sapiens/${rel}_GRCh38/info.txt" ] || { echo "[OK] VEP ${rel} cache already present."; continue; } + peak=56 + [ "$rel" != "$PCGR_VEP_CACHE_RELEASE" ] || peak=52 + if clinical_disk_ok "$dir" "$peak" "VEP ${rel} cache"; then + if ! install_vep_cache "$dir" "$rel"; then + echo "[WARN] Could not install VEP ${rel}. Run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\" (interrupted downloads resume)." + if [ "$rel" = "$VEP_CACHE_RELEASE" ]; then + echo " To retry only this cache: ./scripts/setup.sh --vep-cache \"${GENOME_DIR}\"" + fi + fi + fi + fi + done +} + case "$OPT_IN" in ancestry-panel) install_ancestry_panel; exit $? ;; ancestry-fine-panel) install_ancestry_fine_panel; exit $? ;; @@ -2090,6 +2135,7 @@ case "$OPT_IN" in megane-kmers) ensure_image MEGANE_IMAGE 50 && install_megane_kmers; exit $? ;; mocha-data) install_mocha_data; exit $? ;; locityper-data) install_locityper_data; exit $? ;; + clinical-data) install_clinical_data; exit $? ;; vep-cache) install_vep_cache_opt_in; exit $? ;; annotsv-data) install_annotsv_data; exit $? ;; archaic) install_archaic; exit $? ;; @@ -2459,47 +2505,15 @@ echo "=== Phase 5: AnnotSV annotation data (~5.3 GB + ~1.7 GB download, ~20 GB u install_annotsv_data || true ############################################################################### -# Phase 6: Optional Downloads (instructions only) +# Phase 6: Clinical reference data (default) ############################################################################### echo "" -echo "=== Phase 6: Optional Downloads (manual) ===" -echo "" -echo "The following are only needed for specific steps and are large downloads:" -echo "" - -VEPDIR="${GENOME_DIR}/vep_cache" -if [ -f "${VEPDIR}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38/info.txt" ]; then - echo "[OK] VEP ${VEP_CACHE_RELEASE} cache already present." -else - echo "[SKIP] VEP ${VEP_CACHE_RELEASE} cache (~26 GB) — without it a run skips VEP, vcfanno, the clinical filter and slivar" - echo " Install it before the first run (downloads, checks and unpacks it; ~30 GB unpacked):" - echo " ./scripts/setup.sh --vep-cache ${GENOME_DIR}" -fi -echo "" - -PCGRDIR="${GENOME_DIR}/pcgr_data" -if [ -d "${PCGRDIR}/${PCGR_DATA_BUNDLE}/data" ]; then - echo "[OK] PCGR/CPSR ref data bundle already present." -else - echo "[SKIP] PCGR/CPSR ref data (~7 GB download) — needed for step 17 (cancer predisposition)" - echo " Download:" - echo " mkdir -p ${PCGRDIR} && cd ${PCGRDIR}" - echo " curl -fL -C - -O https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" - echo " tar xzf pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" - echo " mkdir -p ${PCGR_DATA_BUNDLE} && mv data/ ${PCGR_DATA_BUNDLE}/" -fi -echo "" - -# CPSR runs the VEP release inside the PCGR image, not the one step 13 uses. -if [ -f "${VEPDIR}/homo_sapiens/${PCGR_VEP_CACHE_RELEASE}_GRCh38/info.txt" ]; then - echo "[OK] VEP ${PCGR_VEP_CACHE_RELEASE} cache (for CPSR) already present." +echo "=== Phase 6: Clinical reference data ===" +if $CLINICAL_DATA; then + install_clinical_data else - echo "[SKIP] VEP ${PCGR_VEP_CACHE_RELEASE} cache (~24 GB) — needed for step 17 (CPSR runs VEP ${PCGR_VEP_CACHE_RELEASE} inside the PCGR image)" - echo " This is separate from the release-${VEP_CACHE_RELEASE} cache used by step 13. Both coexist in vep_cache/." - echo " Download:" - echo " mkdir -p ${VEPDIR}" - echo " curl -fL -C - -o ${VEPDIR}/$(basename "$(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")") $(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")" - echo " tar xzf ${VEPDIR}/$(basename "$(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")") -C ${VEPDIR}" + echo "[SKIP] Clinical data (--no-clinical-data): steps 13, 30, 23, 31 and 17 need the caches or bundle." + echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" fi ############################################################################### @@ -2520,4 +2534,4 @@ echo " 2. Run: export GENOME_DIR=${GENOME_DIR}" echo " 3. Run: ./scripts/validate-setup.sh " echo " 4. Run: ./scripts/run-all.sh " echo "" -echo "For optional VEP and CPSR setup, see the instructions above." +echo "Review any warnings above; use --clinical-data to retry missing clinical downloads." diff --git a/scripts/validate-setup.sh b/scripts/validate-setup.sh index e8c19b6e4..60beff37c 100755 --- a/scripts/validate-setup.sh +++ b/scripts/validate-setup.sh @@ -588,13 +588,14 @@ else echo " Install it: ./scripts/setup.sh --sample-qc-data ${GENOME_DIR}" fi - # --- VEP cache of the VEP image's release, for step 13 (optional) --- + # --- Clinical data installed by default by setup.sh --- VEP_DIR="${GENOME_DIR}/vep_cache/homo_sapiens" if [ -f "${VEP_DIR}/${VEP_CACHE_RELEASE}_GRCh38/info.txt" ]; then pass "VEP ${VEP_CACHE_RELEASE} cache (step 13): present" else warn "VEP ${VEP_CACHE_RELEASE} cache not found at: ${VEP_DIR}/${VEP_CACHE_RELEASE}_GRCh38" - echo " Required for step 13 (VEP annotation), which downloads it (~26 GB) the first time it runs." + echo " Missing cache skips steps 13 (VEP), 30 (vcfanno), 23 (clinical filter) and 31 (slivar)." + echo " Install it: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" echo " See docs/00-reference-setup.md for full instructions." fi @@ -604,7 +605,7 @@ else else warn "VEP ${PCGR_VEP_CACHE_RELEASE} cache not found at: ${VEP_DIR}/${PCGR_VEP_CACHE_RELEASE}_GRCh38" echo " Required for step 17 (CPSR). ${PCGR_IMAGE} needs VEP ${PCGR_VEP_CACHE_RELEASE}, separate from step 13's VEP ${VEP_CACHE_RELEASE}." - echo " curl -fL -C - -O $(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")" + echo " Missing cache skips step 17. Install it: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" echo " See docs/17-cpsr.md for full instructions." fi @@ -614,10 +615,8 @@ else pass "PCGR/CPSR ref data bundle: present" else warn "PCGR ref data bundle not found at: ${PCGR_DIR}" - echo " Required for step 17 (CPSR cancer predisposition). Download ~7 GB:" - echo " cd ${GENOME_DIR}/pcgr_data" - echo " curl -fL -C - -O https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" - echo " tar xzf pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz && mkdir -p ${PCGR_DATA_BUNDLE} && mv data/ ${PCGR_DATA_BUNDLE}/" + echo " Missing bundle skips step 17 (CPSR cancer predisposition)." + echo " Install it: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" echo " See docs/17-cpsr.md for full instructions." fi diff --git a/tests/fake-docker/base-setup.sh b/tests/fake-docker/base-setup.sh index 6e5412647..82c7ee9dd 100755 --- a/tests/fake-docker/base-setup.sh +++ b/tests/fake-docker/base-setup.sh @@ -4,6 +4,7 @@ # finish with exit 0. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" +seed_clinical_downloads G="${CASE_WORK}/genome" mkdir -p "$G" diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh index 4c289cdde..83a674b03 100644 --- a/tests/fake-docker/setup-clinical-default.sh +++ b/tests/fake-docker/setup-clinical-default.sh @@ -2,6 +2,7 @@ # Default clinical downloads, explicit opt-out, disk gates and corrupt input. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" +seed_clinical_downloads export FETCH_TRIES=1 FETCH_WAIT=0 G="${CASE_WORK}/without" run_expect 0 without "${SCRIPTS}/setup.sh" --no-clinical-data "$G" @@ -17,6 +18,16 @@ for rel in 116 115; do done [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'PCGR bundle missing' output_lacks clinical 'Phase 1:|Docker Images' +# Installed clinical data turns the three default launch-plan rows on. +seed_reference "$G" +seed_clinvar "$G" +seed_sample "$G" sample1 +use_output_hook +GENOME_DIR="$G" SKIP_VALIDATION=true run_expect 0 plan "${SCRIPTS}/run-all.sh" sample1 male +output_has plan '^ 13 VEP +runs$' +output_has plan '^ 23 Clinical filter +runs$' +output_has plan '^ 17 CPSR +runs$' + # A second install does no network work, even with too little space. cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" diff --git a/versions.env b/versions.env index 018f5c605..5c9595f9d 100644 --- a/versions.env +++ b/versions.env @@ -192,6 +192,7 @@ PCGR_VEP_CACHE_RELEASE="115" PCGR_DATA_BUNDLE="20260620" # Exact archive size from the server; SHA-256 is measured from the full download. PCGR_DATA_BYTES="7664082467" +PCGR_DATA_SHA256="f9b8149e37c7e63bd7d877713cdb11a1a609c492b3f1a055fb7e90cd80273e54" # pypgx-bundle git tag (step 32): must equal the version of PYPGX_IMAGE. PYPGX_BUNDLE_VERSION="0.26.0" # Cyrius (step 21, opt-in; legacy: no automated bumps) is installed by From 7be974dbfeeee0ba1639bf2e09c38eeb82a3c122 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:27:04 +0100 Subject: [PATCH 61/86] test(ancestry): use a verb no plan adds for the unknown-verb case og ancestry-fine paint is now a planned verb, so the test of the unknown-verb refusal names one that never will be. Bead: D14 --- rust/crates/og-ancestry/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 5ae862961..743ea611a 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -730,9 +730,9 @@ mod tests { #[test] fn unknown_verbs_and_missing_options_are_usage_errors() { - let (rc, _, err) = testutil::run(&["paint"]); + let (rc, _, err) = testutil::run(&["no-such-verb"]); assert_eq!(rc, 2); - assert!(err.contains("invalid choice: 'paint'"), "{err}"); + assert!(err.contains("invalid choice: 'no-such-verb'"), "{err}"); let (rc, _, err) = testutil::run(&["order", "--pop", "x"]); assert_eq!(rc, 2); assert!( From 4fb1beaaac40ddaef02e35a4268f438ba791bac1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:27:29 +0100 Subject: [PATCH 62/86] fix(bloodtype): read every record that covers a marker, in any order The first record at a position won, so an all-sites reference record or a split SNV record before the ABO c.261 insertion made the site reference and ABO read O. Each record now gives a reading with a weight (a call of the marker ALT, a no call of a record naming it, a reference call, any other no call) and the strongest counts, so order does not matter. A reference-only record shorter than a deletion marker says nothing, so the reads decide. A record that spans the marker from upstream with an allele that changes its base makes the site a no call. The index path now keeps the records that span a position too. Bead: trait-40 Bead: trait-41 Bead: trait-29 --- rust/crates/og-bloodtype/src/genotypes.rs | 396 ++++++++++++++++++---- rust/crates/og-bloodtype/tests/report.rs | 18 +- 2 files changed, 344 insertions(+), 70 deletions(-) diff --git a/rust/crates/og-bloodtype/src/genotypes.rs b/rust/crates/og-bloodtype/src/genotypes.rs index a04dfbee2..6eac1aec9 100644 --- a/rust/crates/og-bloodtype/src/genotypes.rs +++ b/rust/crates/og-bloodtype/src/genotypes.rs @@ -6,14 +6,30 @@ //! index: a few hundred BGZF blocks instead of the whole file. Without a //! usable index (a plain or gzip file, no index, an index older than the //! VCF, or one that cannot be read) the file is read once, line by line, -//! and a line is split only when its CHROM and POS are a marker's. Both +//! and a line is split only when its REF covers a marker's position. Both //! ways apply the same records in the same order, so they give the same //! sites. +//! +//! Several records can cover one site: an all-sites VCF writes a reference +//! record and then the indel at one position, a split multi-allelic record +//! is two lines, and a deletion that starts upstream spans the site. Every +//! one of them is read, and the answer does not depend on their order: +//! +//! - a record with a non-reference allele that changes the site's base +//! from upstream makes the site a no call, whatever the site's own +//! records say; +//! - else, of the records at the site's position, a call of the marker's +//! ALT wins, then a no call of a record that names the marker's ALT, +//! then a reference call, then any other no call; the first of equals +//! counts, and another allele a losing call names stays in the note; +//! - a reference-only record whose REF is shorter than the marker's (an +//! all-sites record or a gVCF block starting at a deletion marker) says +//! nothing past its own bases, so the site reads as if it had no record. use crate::Error; use crate::markers::{Marker, Sites}; use indexmap::IndexMap; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::Path; /// Reads (MAPQ >= 20) before a site missing from the VCF counts as reference. @@ -97,11 +113,11 @@ pub fn norm_chrom(c: &str) -> String { } /// The sites at each (chrom, pos), in table order. -struct ByPos(HashMap>>); +struct ByPos(HashMap>>); impl ByPos { fn new(sites: &Genotyped) -> ByPos { - let mut m: HashMap>> = HashMap::new(); + let mut m: HashMap>> = HashMap::new(); for (i, s) in sites.values().enumerate() { m.entry(s.row.chrom.clone()) .or_default() @@ -136,10 +152,108 @@ fn check_header(line: &str, path: &str) -> Result<(), Error> { Ok(()) } -/// Applies one record line (no line ending) to the sites at its position. -fn apply_record(line: &str, by_pos: &ByPos, sites: &mut Genotyped) { - let mut c = line.splitn(3, '\t'); - let (Some(chrom), Some(pos), Some(_)) = (c.next(), c.next(), c.next()) else { +/// How much a record at a site's own position says about the marker's +/// ALT; the strongest one read counts. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Weight { + /// A no call of a record that does not name the marker's ALT. + Other, + /// A call with no copy of the marker's ALT. + Reference, + /// A no call of a record that names the marker's ALT (`./.`, a + /// failed FILTER): the ALT may be there. + Doubt, + /// A call with the marker's ALT. + Alt, +} + +/// What one record at a site's position says: status, alt copies, note. +#[derive(Debug, Clone, PartialEq)] +struct Reading { + weight: Weight, + status: Status, + n: Option, + note: String, +} + +impl Reading { + fn no_call(weight: Weight, note: String) -> Reading { + Reading { + weight, + status: Status::NoCall, + n: None, + note, + } + } +} + +/// Everything the records said about one site so far. +#[derive(Debug, Default)] +struct Evidence { + /// The strongest reading at the site's position. + best: Option, + /// The notes of losing calls (another allele at the position). + others: Vec, + /// The first upstream record that changes the site's base. + span: Option, +} + +impl Evidence { + fn add(&mut self, r: Reading) { + let loser = match &mut self.best { + Some(b) if r.weight > b.weight => Some(std::mem::replace(b, r)), + Some(_) => Some(r), + None => { + self.best = Some(r); + None + } + }; + if let Some(l) = loser + && l.status == Status::Called + && !l.note.is_empty() + && !self.others.contains(&l.note) + { + self.others.push(l.note); + } + } + + /// The site as the records leave it; `Absent` when none said anything. + fn settle(self, site: &mut Site) { + if let Some(note) = self.span { + site.status = Status::NoCall; + site.n = None; + site.note = note; + } else if let Some(b) = self.best { + site.status = b.status; + site.n = b.n; + site.note = if b.status == Status::Called { + let others = self.others.into_iter().filter(|o| *o != b.note); + std::iter::once(b.note.clone()) + .chain(others) + .filter(|p| !p.is_empty()) + .collect::>() + .join("; ") + } else { + b.note + }; + } + } +} + +/// Applies one record line (no line ending) to the sites it covers: those +/// at its position and those its REF spans. `only` keeps the sites at one +/// position (the index path reads each position's records on their own). +fn apply_record( + line: &str, + by_pos: &ByPos, + sites: &Genotyped, + ev: &mut [Evidence], + only: Option, +) { + let mut c = line.splitn(5, '\t'); + let (Some(chrom), Some(pos), Some(_), Some(rf), Some(_)) = + (c.next(), c.next(), c.next(), c.next(), c.next()) + else { return; }; let chrom = if chrom.starts_with("chr") { @@ -150,14 +264,22 @@ fn apply_record(line: &str, by_pos: &ByPos, sites: &mut Genotyped) { let Some(here) = by_pos.0.get(chrom.as_ref()) else { return; }; - let Some(names) = py_pos(pos).and_then(|p| here.get(&p)) else { + let Some(p) = py_pos(pos) else { return; }; + let span = i64::try_from(rf.len().max(1)).unwrap_or(1); + let mut hits = here + .range(p..p.saturating_add(span)) + .filter(|(at, _)| only.is_none_or(|o| o == **at)) + .peekable(); + if hits.peek().is_none() { + return; + } let f: Vec<&str> = line.split('\t').collect(); if f.len() < 10 { return; } - let (rf, alts, filt) = (f[3], f[4].split(',').collect::>(), f[6]); + let (alts, filt) = (f[4].split(',').collect::>(), f[6]); let fmt: Vec<&str> = f[8].split(':').collect(); let smp: Vec<&str> = f[9].split(':').collect(); let gt = fmt @@ -167,18 +289,57 @@ fn apply_record(line: &str, by_pos: &ByPos, sites: &mut Genotyped) { .copied() .unwrap_or("."); let alleles: Vec<&str> = gt.split(['/', '|']).collect(); - for &i in names { - let site = &mut sites[i]; - if site.status == Status::Called { - continue; // a second record at the position: the first one counts + for (&at, names) in hits { + if at == p { + for &i in names { + if let Some(r) = read_record(&sites[i].row, rf, &alts, filt, &alleles) { + ev[i].add(r); + } + } + continue; + } + let offset = usize::try_from(at - p).unwrap_or(0); + if changes_base(rf, &alts, &alleles, offset) { + for &i in names { + ev[i].span.get_or_insert_with(|| { + format!( + "a record at {}:{} ({rf}>{}, genotype {}) spans this site", + f[0], + f[1], + f[4], + alleles.join("/") + ) + }); + } } - genotype_site(site, rf, &alts, filt, &alleles); } } -/// The site's status, alt copies and note from one record. -pub fn genotype_site(site: &mut Site, rf: &str, alts: &[&str], filt: &str, alleles: &[&str]) { - let r = &site.row; +/// Whether a record that starts `offset` bases before a site and spans it +/// calls an allele that changes the site's base: one of another length +/// (an indel, a symbolic or `*` allele) or with another base there. A +/// reference or missing genotype changes nothing. +fn changes_base(rf: &str, alts: &[&str], alleles: &[&str], offset: usize) -> bool { + alleles.iter().any(|a| { + let Some(k) = a.parse::().ok().filter(|&k| k > 0) else { + return false; + }; + alts.get(k - 1).is_none_or(|alt| { + alt.len() != rf.len() || alt.as_bytes().get(offset) != rf.as_bytes().get(offset) + }) + }) +} + +/// What one record at the site's position says about it; `None` when it +/// says nothing: a reference-only record whose REF is shorter than the +/// marker's, which reads no base past its own. +fn read_record( + r: &Marker, + rf: &str, + alts: &[&str], + filt: &str, + alleles: &[&str], +) -> Option { // A null row may list several ALTs (null alleles at one position): any of them counts. let wants: Vec<&str> = if r.ref_ == rf { r.alt.split(',').collect() @@ -191,42 +352,58 @@ pub fn genotype_site(site: &mut Site, rf: &str, alts: &[&str], filt: &str, allel .filter(|(_, a)| wants.contains(a)) .map(|(i, _)| (i + 1).to_string()) .collect(); + let doubt = if idx.is_empty() { + Weight::Other + } else { + Weight::Doubt + }; let joined = alleles.join("/"); if alleles.contains(&".") || alleles.len() != 2 { - site.status = Status::NoCall; - site.note = format!("genotype {joined}"); - return; + return Some(Reading::no_call(doubt, format!("genotype {joined}"))); } - // A record whose REF is not the marker's REF says nothing about the - // marker's ALT, unless it is a reference call over a span that starts - // with the marker's REF. - if rf != r.ref_ && !(rf.starts_with(r.ref_.as_str()) && alleles.iter().all(|a| *a == "0")) { - site.note = format!("REF {rf} at this position, the marker's REF is {}", r.ref_); - site.status = Status::NoCall; - return; + let all_ref = alleles.iter().all(|a| *a == "0"); + if rf != r.ref_ { + if all_ref && rf.len() < r.ref_.len() && r.ref_.starts_with(rf) { + return None; + } + // A record whose REF is not the marker's REF says nothing about + // the marker's ALT, unless it is a reference call over a span that + // starts with the marker's REF. + if !(all_ref && rf.starts_with(r.ref_.as_str())) { + return Some(Reading::no_call( + Weight::Other, + format!("REF {rf} at this position, the marker's REF is {}", r.ref_), + )); + } } - if filt != "PASS" && filt != "." && alleles.iter().any(|a| *a != "0") { - site.status = Status::NoCall; - site.note = format!("FILTER {filt}"); - return; + if filt != "PASS" && filt != "." && !all_ref { + return Some(Reading::no_call(doubt, format!("FILTER {filt}"))); } - site.status = Status::Called; - site.n = Some( - alleles - .iter() - .filter(|a| idx.iter().any(|i| i == *a)) - .count() as i64, - ); - site.note.clear(); - if alleles + let n = alleles + .iter() + .filter(|a| idx.iter().any(|i| i == *a)) + .count() as i64; + let note = if alleles .iter() .any(|a| *a != "0" && !idx.iter().any(|i| i == a)) { - site.note = format!( + format!( "another allele at this position ({rf}>{}, genotype {joined})", alts.join(",") - ); - } + ) + } else { + String::new() + }; + Some(Reading { + weight: if n > 0 { + Weight::Alt + } else { + Weight::Reference + }, + status: Status::Called, + n: Some(n), + note, + }) } /// Genotypes the sites from the sample's VCF (its one sample column). @@ -236,10 +413,12 @@ pub fn read_vcf(path: &str, markers: Sites) -> Result<(Genotyped, Access), Error .map(|(k, m)| (k, Site::new(m))) .collect(); let by_pos = ByPos::new(&sites); + let mut ev: Vec = sites.values().map(|_| Evidence::default()).collect(); if let Some(records) = indexed_records(path, &sites)? { - for line in &records { - apply_record(line, &by_pos, &mut sites); + for (pos, line) in &records { + apply_record(line, &by_pos, &sites, &mut ev, Some(*pos)); } + settle(&mut sites, ev); return Ok((sites, Access::Index)); } let mut lines = og_io::Lines::new( @@ -261,8 +440,9 @@ pub fn read_vcf(path: &str, markers: Sites) -> Result<(Genotyped, Access), Error header = true; continue; } - apply_record(l, &by_pos, &mut sites); + apply_record(l, &by_pos, &sites, &mut ev, None); } + settle(&mut sites, ev); if !header { return Err(Error(format!( "{path} has no #CHROM header line: not a VCF" @@ -271,10 +451,18 @@ pub fn read_vcf(path: &str, markers: Sites) -> Result<(Genotyped, Access), Error Ok((sites, Access::Stream)) } -/// The record lines at the marker positions, read through the VCF's -/// index, those of each position in file order; `None` when the file is to +/// Each site takes what its records said. +fn settle(sites: &mut Genotyped, ev: Vec) { + for (s, e) in sites.values_mut().zip(ev) { + e.settle(s); + } +} + +/// The record lines that cover each marker position (start there or span +/// it), read through the VCF's index, with the position they were read +/// for, those of each position in file order; `None` when the file is to /// be read whole. The `#CHROM` line is checked as a full pass checks it. -fn indexed_records(path: &str, sites: &Genotyped) -> Result>, Error> { +fn indexed_records(path: &str, sites: &Genotyped) -> Result>, Error> { let Some(mut reader) = usable_index(path) else { return Ok(None); }; @@ -319,15 +507,14 @@ fn indexed_records(path: &str, sites: &Genotyped) -> Result>, let Ok(raw) = got else { return Ok(None); }; - let l = String::from_utf8_lossy(&raw).into_owned(); - if l.split('\t').nth(1).and_then(py_pos) == Some(*pos) { - out.push(l); - } + // The index also returns a record whose INFO/END reaches + // the position; apply_record keeps those whose REF does. + out.push((*pos, String::from_utf8_lossy(&raw).into_owned())); } } } - // A site reads only the records at its own position, and those come in - // file order, so the order between positions does not matter. + // A site reads only the records that cover its own position, and those + // come in file order, so the order between positions does not matter. Ok(Some(out)) } @@ -421,6 +608,23 @@ mod tests { }) } + /// The site as these records, in this order, leave it. + fn genotype(s: &mut Site, rf: &str, alts: &[&str], filt: &str, alleles: &[&str]) { + records(s, &[(rf, alts, filt, alleles)]); + } + + type Rec<'a> = (&'a str, &'a [&'a str], &'a str, &'a [&'a str]); + + fn records(s: &mut Site, recs: &[Rec]) { + let mut e = Evidence::default(); + for (rf, alts, filt, alleles) in recs { + if let Some(r) = read_record(&s.row, rf, alts, filt, alleles) { + e.add(r); + } + } + e.settle(s); + } + #[test] fn chroms_normalise() { assert_eq!(norm_chrom("chr7"), "chr7"); @@ -432,7 +636,7 @@ mod tests { #[test] fn a_het_call_counts_one_alt_copy() { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "PASS", &["0", "1"]); + genotype(&mut s, "G", &["A"], "PASS", &["0", "1"]); assert_eq!( (s.status, s.n, s.note.as_str()), (Status::Called, Some(1), "") @@ -442,7 +646,7 @@ mod tests { #[test] fn the_second_alt_of_a_multiallelic_record() { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["C", "A"], "PASS", &["1", "2"]); + genotype(&mut s, "G", &["C", "A"], "PASS", &["1", "2"]); assert_eq!((s.status, s.n), (Status::Called, Some(1))); assert_eq!( s.note, @@ -453,24 +657,24 @@ mod tests { #[test] fn a_filtered_alt_call_is_no_call_but_a_refcall_is_reference() { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "LowQual", &["1", "1"]); + genotype(&mut s, "G", &["A"], "LowQual", &["1", "1"]); assert_eq!( (s.status, s.note.as_str()), (Status::NoCall, "FILTER LowQual") ); let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "RefCall", &["0", "0"]); + genotype(&mut s, "G", &["A"], "RefCall", &["0", "0"]); assert_eq!((s.status, s.n), (Status::Called, Some(0))); } #[test] fn another_ref_is_no_call_unless_a_reference_span() { let mut s = site("G", "A"); - genotype_site(&mut s, "C", &["T"], "PASS", &["1", "1"]); + genotype(&mut s, "C", &["T"], "PASS", &["1", "1"]); assert_eq!(s.status, Status::NoCall); assert_eq!(s.note, "REF C at this position, the marker's REF is G"); let mut s = site("G", "A"); - genotype_site(&mut s, "GT", &["G"], "PASS", &["0", "0"]); + genotype(&mut s, "GT", &["G"], "PASS", &["0", "0"]); assert_eq!((s.status, s.n), (Status::Called, Some(0))); } @@ -478,11 +682,77 @@ mod tests { fn a_haploid_or_missing_genotype_is_no_call() { for gt in [&["1"][..], &[".", "."], &["0", "1", "1"]] { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "PASS", gt); + genotype(&mut s, "G", &["A"], "PASS", gt); assert_eq!(s.status, Status::NoCall, "{gt:?}"); } } + #[test] + fn the_strongest_record_at_a_position_counts_in_any_order() { + let r: Rec = ("T", &["."], ".", &["0", "0"]); + let ins: Rec = ("T", &["TC"], "PASS", &["0", "1"]); + let snv: Rec = ("T", &["G"], "PASS", &["1", "0"]); + let ins_nocall: Rec = ("T", &["TC"], "PASS", &[".", "."]); + let other_ref: Rec = ("TA", &["T"], "PASS", &["0", "1"]); + for (recs, want) in [ + (vec![r, ins], (Status::Called, Some(1), "")), + (vec![ins, r], (Status::Called, Some(1), "")), + ( + vec![snv, ins], + ( + Status::Called, + Some(1), + "another allele at this position (T>G, genotype 1/0)", + ), + ), + (vec![r, ins_nocall], (Status::NoCall, None, "genotype ./.")), + (vec![ins_nocall, r], (Status::NoCall, None, "genotype ./.")), + (vec![other_ref, r], (Status::Called, Some(0), "")), + ( + vec![other_ref], + ( + Status::NoCall, + None, + "REF TA at this position, the marker's REF is T", + ), + ), + (vec![ins, ins], (Status::Called, Some(1), "")), + ] { + let mut s = site("T", "TC"); + records(&mut s, &recs); + assert_eq!((s.status, s.n, s.note.as_str()), want, "{recs:?}"); + } + } + + #[test] + fn a_shorter_reference_record_says_nothing() { + let mut s = site("CG", "C"); + genotype(&mut s, "C", &["."], ".", &["0", "0"]); + assert_eq!((s.status, s.n), (Status::Absent, None)); + let mut s = site("CG", "C"); + genotype(&mut s, "C", &["<*>"], ".", &["0", "0"]); + assert_eq!(s.status, Status::Absent); + let mut s = site("CG", "C"); + genotype(&mut s, "C", &["T"], "PASS", &["0", "1"]); + assert_eq!(s.status, Status::NoCall); + } + + #[test] + fn a_spanning_allele_changes_the_base_when_its_length_or_base_differs() { + // CG at the record, the site at offset 1 (the G). + assert!(changes_base("CG", &["CA"], &["1", "1"], 1)); + assert!(changes_base("CG", &["C"], &["0", "1"], 1)); + assert!(changes_base("CG", &["C", "CA"], &["1", "2"], 1)); + assert!(changes_base("CG", &["*"], &["0", "1"], 1)); + assert!( + changes_base("CG", &["A"], &["0", "2"], 1), + "an ALT index the record lacks" + ); + assert!(!changes_base("CG", &["TG"], &["1", "1"], 1)); + assert!(!changes_base("CG", &["C"], &["0", "0"], 1)); + assert!(!changes_base("CG", &["C"], &[".", "."], 1)); + } + #[test] fn depth_decides_a_site_with_no_record() { let mut g: Genotyped = IndexMap::new(); diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index 893169aa8..6cb81053f 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -249,10 +249,7 @@ fn both_ways(d: &Path, records: &[String]) -> (PathBuf, PathBuf) { } at = end; } - let names: Vec = seqs - .iter() - .flat_map(|s| s.0.bytes().chain([0])) - .collect(); + let names: Vec = seqs.iter().flat_map(|s| s.0.bytes().chain([0])).collect(); let mut tbi: Vec = b"TBI\x01".to_vec(); let i32s = |v: &mut Vec, xs: &[i32]| xs.iter().for_each(|x| v.extend(x.to_le_bytes())); // n_ref, VCF preset, columns 1 2 0, meta '#', skip 0, names. @@ -328,7 +325,10 @@ fn a_later_record_of_the_marker_allele_that_is_no_call_wins_over_a_reference_rec let d = dir("abo261-nocall"); let r = line("chr9", 133257521, "T", ".", ".", "0/0"); for (later, note) in [ - (line("chr9", 133257521, "T", "TC", "PASS", "./."), "genotype ./."), + ( + line("chr9", 133257521, "T", "TC", "PASS", "./."), + "genotype ./.", + ), ( line("chr9", 133257521, "T", "TC", "LowQual", "0/1"), "FILTER LowQual", @@ -388,7 +388,11 @@ fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { // KEL c.578C>T (K) is chr7:142957921 G>A. A padded record one base // upstream carries the same change (CG>CA), or a deletion of that // base with it (DeepVariant's CG>C,CA 1/2). - for (r, alt, gt) in [("CG", "CA", "1/1"), ("CG", "C,CA", "1/2"), ("CG", "C", "0/1")] { + for (r, alt, gt) in [ + ("CG", "CA", "1/1"), + ("CG", "C,CA", "1/2"), + ("CG", "C", "0/1"), + ] { let recs = [line("chr7", 142957920, r, alt, "PASS", gt)]; let (status, n, note) = site_both_ways(&d, &recs, "KEL_578"); assert_eq!( @@ -401,7 +405,7 @@ fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { ) ); let t = run(&d, &recs, reads()); - assert_eq!(t["Kell"][0], "K/k not determined", "{recs:?}"); + assert!(t["Kell"][0].starts_with("K/k not determined"), "{recs:?}"); assert_ne!(t["Kell"][2], "high", "{recs:?}"); } // Split (bcftools norm -m-any): the deletion before the SNV record From f446ec41265d70af97bc19a9e59b9937cad264a9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:27:30 +0100 Subject: [PATCH 63/86] docs(bloodtype): say how overlapping records at a marker are read Bead: trait-40 Bead: trait-41 Bead: trait-29 --- CHANGELOG.md | 7 +++++++ docs/41-bloodtype.md | 13 ++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..ab677d5bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 41 (blood groups) reads every VCF record that covers a marker site, and their order no longer changes the answer. Before, the first record at a position won. An all-sites VCF writes a reference record and then the indel at one position (`T . 0/0`, then `T TC 0/1` at ABO c.261), so a person with an A or B allele read O. A split multi-allelic record with the SNV first did the same. No column, file or option changes; only the values of the affected sites do. What changes: + - A call of the marker's ALT now wins over the other records at the position. Another allele a second record calls there is named in the note, as when one record holds both. + - A no call of a record that names the marker's ALT (`./.`, or a failed FILTER) now wins over a reference record at the same position, so the site is a no call instead of a confident reference. + - A 0/0 record over a REF shorter than the marker's (an all-sites `C . 0/0`, or a gVCF block that starts at the position) now counts as no record, so the reads decide. Before it was a no call: ABO read "A, A1 or A2 not determined", and Vel, HPA-14 and every system with a deletion marker lost confidence. + - A record that starts upstream and spans a marker with an allele that changes the marker's base (a deletion over it, or a padded `CG CA`, as DeepVariant writes `CG C,CA` when a deletion and an SNV overlap) now makes the site a no call, with a note that names the record. Before, the site had no record at its own position and 8 reads made it a confident reference: a K/K person read k/k at high confidence. + - A VCF with one record per position and no overlapping records, as DeepVariant writes, gives the same output as before: the 34 parity cases still agree with the Python oracle byte for byte. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 20f2cb000..2456d51b5 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -33,7 +33,7 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, left-aligned records (DeepVariant writes them so; split multi-allelic records are read as they are) +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, left-aligned records (DeepVariant writes them so). Split multi-allelic records, all-sites VCFs (a reference record and then the variant at one position) and gVCF reference blocks are read too: every record that covers a site counts, whatever their order (see below) - `${SAMPLE}/aligned/${SAMPLE}_sorted.bam` with its `.bai` (`ALIGN_DIR` as for the other BAM steps). Optional: without it RhD is not determined, and a site the VCF has no record for is read as the reference allele at confidence medium. ## Command @@ -62,7 +62,14 @@ With `run-all.sh`: `TOOLS=...,bloodtype`. It is opt-in and needs no data, so a r | HPA-7 to -14, -16 to -35 | one variant each in ITGB3, ITGA2B, ITGA2, GP1BB or GP9 | one allele per copy; the a antigen is on the reference allele, the low-prevalence b antigen on the other | | HNA-3, -4, -5 | SLC44A2 c.461G>A, ITGAM c.230G>A, ITGAL c.2372G>C | one allele per copy; GRCh38 carries HNA-3b | -A site the VCF has no record for is the reference allele when the BAM has at least 8 reads there at MAPQ 20, and not determined below that. Without the BAM it is read as reference, and the confidence says so. The step leaves out of `samtools bedcov` the rows on a contig the BAM has no `@SQ` line for (a BAM cut to a few chromosomes), because `bedcov` would stop on them. Those sites then have no depth row: the ones with no VCF record are not determined, and RhD is not determined without chr1. A record at a site whose REF is not the table's REF is a no call, because it says nothing about the table's ALT; the one exception is a 0/0 call over a longer REF that starts with the table's REF, which is the reference allele. +A site the VCF has no record for is the reference allele when the BAM has at least 8 reads there at MAPQ 20, and not determined below that. Without the BAM it is read as reference, and the confidence says so. The step leaves out of `samtools bedcov` the rows on a contig the BAM has no `@SQ` line for (a BAM cut to a few chromosomes), because `bedcov` would stop on them. Those sites then have no depth row: the ones with no VCF record are not determined, and RhD is not determined without chr1. A record at a site whose REF is not the table's REF is a no call, because it says nothing about the table's ALT. There are two exceptions. A 0/0 call over a longer REF that starts with the table's REF is the reference allele. A 0/0 record over a shorter REF (an all-sites VCF's `C . 0/0`, or a gVCF block that starts at a deletion marker such as ABO c.1061delC) says nothing past its own base, so the site reads as one with no record: the reads decide. + +When several records cover one site, each one is read and their order does not matter: + +- A call of the table's ALT wins over the other records at the site's position. An all-sites VCF writes `T . 0/0` and then `T TC 0/1` at ABO c.261, so the insertion counts. Another allele that a second record calls there (a split `T G 1/0`) is named in the note. +- A no call of a record that names the table's ALT (`./.`, or a failed FILTER) comes next: the ALT may be there, so a reference record beside it does not make the site reference. +- Then a reference call, then any other no call. +- A record that starts upstream and spans the site with an allele that changes the site's base (a deletion over it, or a padded record such as `CG CA`) makes the site a no call, whatever the site's own records say. A reference call or a no call upstream changes nothing, and neither does an allele that keeps the site's base. ## Output @@ -94,7 +101,7 @@ Lacking an antigen is common (most people lack HPA-1b, for example). It means a ## Runtime -Not measured on a whole genome. The step reads the VCF once, line by line, and samtools reads the BAM at 284 small regions. +Not measured on a whole genome. The step reads the marker positions through the VCF's index (the VCF once, line by line, without one), and samtools reads the BAM at 284 small regions. ## Notes From 875196f849f2b6e40bea7637af672e2e45b016e5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:37:19 +0100 Subject: [PATCH 64/86] test(live): inspect alignment rendering inside the viewer shadow root Bead: pgp-iqon.11 --- tests/test_live_viewer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 803d7e33f..df50d0d61 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -272,9 +272,13 @@ def loaded(): check("map: moving updates the same iframe to the map's locus", b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) check("viewer: alignments remain visible at 49 kb", - b.wait("(function(){var v=savedFrame.contentDocument.querySelector('.igv-viewport[data-track-type=alignment]');" + b.wait("(function(){var h=savedFrame.contentDocument.getElementById('igv');" + "var v=h && h.shadowRoot && h.shadowRoot.querySelector('.igv-viewport[data-track-type=alignment]');" "var n=v && v.querySelector('.igv-zoom-in-notice-container');" - "return !!(v && v.querySelector('canvas') && n && n.style.display!=='flex');})()")) + "return !!(v && v.querySelector('canvas') && n && n.style.display!=='flex');})()"), + b.eval("JSON.stringify(Array.from(savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelectorAll('.igv-viewport')).map(function(v){" + "var n=v.querySelector('.igv-zoom-in-notice-container');return {type:v.dataset.trackType," + "notice:n && n.style.display,canvas:v.querySelectorAll('canvas').length,text:v.textContent.slice(0,180)};}))")) if os.environ.get("MAP_SCREENS"): os.makedirs(os.environ["MAP_SCREENS"], exist_ok=True) for width, height in ((1440, 900), (390, 844)): From 67856fddf4a13fd2df1808a6a7f76aa309279701 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:38:44 +0100 Subject: [PATCH 65/86] fix(build): mount and fingerprint the embedded imprinting table The supported builder stages rust at /src, so the table needs a read-only mount at /assets. Include its blob in the source stamp so an asset edit makes old binaries stale. Keep the stamp format and release-install checks. The new build checks failed five assertions against the previous builder and all pass with this change. The original missing-mount failure is upstream run 43658012836. Bead: lic-41 Bead: lin-17 --- CHANGELOG.md | 2 +- docs/rust.md | 2 +- rust/README.md | 2 +- rust/og.sh | 35 ++++++++++++++++++++++++++--------- tests/test_og_check.sh | 24 +++++++++++++++++++++--- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78de0ffed..ab95d832e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - ROH summaries now divide F_ROH by the GRCh38 length outside the bands of the assessed autosomes. Per-site rows identify those chromosomes, including chromosomes with no ROH. A new `roh.data.scope` records the chromosomes, denominator and scope source. Relationship labels and UPD flags require sites on all 22 autosomes and the full unmasked autosome length. Segment-only files use their observed chromosomes as an unverified lower bound and get neither label. Empty input has null F_ROH. Segment counts and the step 11 summary table keep their format. -- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. +- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. The container builder mounts it, and the source stamp includes its blob hash, so a table edit makes an old binary stale. The stamp format and `tree-hash` command name stay the same. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. - The ROH collector now streams per-site rows instead of keeping the whole file in memory. On 1.1 million synthetic site rows, peak RSS fell from 65.7-67.0 MB to 3.0-3.5 MB. CPU time stayed near 0.2 seconds. Under shared host load, wall time ranged from 1.40-21.46 seconds before and 7.89-38.16 seconds after; these timings do not establish a speedup. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: diff --git a/docs/rust.md b/docs/rust.md index 0e0de2374..a9da46f20 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -70,7 +70,7 @@ The binary is static, so it runs in any image that has `/bin/sh`: the Python ima `bin/.og` is not in git, so after a pull or an edit it can hold a binary built from other code. Results would then come from code the checkout does not show. To stop that: -- The build stamps the binary with the git tree hash of `rust/`: `git rev-parse HEAD:rust` on a clean checkout, or for an edited `rust/` the tree those edits would commit. Any edit, committed or not, changes the hash. +- The build stamps the binary with a source hash of two git object IDs: the tree of `rust/` as it is on disk and the blob of `assets/imprinted_regions.tsv`. The table is embedded at compile time. Any edit to either input, committed or not, changes the hash. The `tree-hash` verb and stamp format retain their names. - `rust/og.sh check` reads the stamp from each binary with `grep`, without running it, and compares it with the checkout's. The macOS pair is optional, because no pipeline step runs it, but a stale one is refused like a Linux one. - `run-all.sh` (even with `SKIP_VALIDATION=true`), `main.nf` and `validate-setup.sh` run that check. A binary built from another `rust/` stops them with `run scripts/setup.sh --og`. - A missing binary is still a note, which names steps 04, 05, 07, 09b, 21, 25, 32 and 33. Those steps, step 06 with `CLINVAR_SCREEN` and `CLINVAR_CDNA_CHECK`, step 17, and steps 23 and 31 with their modules, stop without it: `bin/og` says `ERROR: og is not built for ; run scripts/setup.sh --og` (exit 127). Step 32 does not read that as an unmeasured share: it stops. `REQUIRED` in `rust/og.sh` stays `false` until the fake-docker and Nextflow stub tests, which start runs without `og`, build it first; then a missing binary stops a run before any step. diff --git a/rust/README.md b/rust/README.md index ce5fae19c..a4fa66be8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -33,7 +33,7 @@ cargo clippy --all-targets -- -D warnings cargo build --release && ../tests/rust/pycompat-diff.sh target/release/og # pycompat against CPython ``` -A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the tree hash the pipeline checks. +A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the source hash of `rust/` and its embedded imprinting table that the pipeline checks. CI readers also run through `og dev pharmcat diplotypes`, `og dev nf-trace requests|attempts` and `og dev smoke`. See [the Rust command notes](../docs/rust.md#ci-readers). `og traits table`, `og traits phase-sites` and `og traits check` read step 42's diff --git a/rust/og.sh b/rust/og.sh index 70f5af058..0df99a92f 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -19,15 +19,15 @@ # --required, or once REQUIRED below is true) no # Linux pair. The macOS pair is optional, since no # pipeline step runs it, but a stale one is refused -# rust/og.sh tree-hash the git tree hash of rust/ as it is on disk +# rust/og.sh tree-hash the source hash of rust/ and its embedded table # rust/og.sh built the tree hash the binaries were built from, for # run_manifest.tsv ("not built" when there are none) # -# The tree hash is git's: `git rev-parse HEAD:rust` on a clean checkout, and -# for an edited rust/ the tree those edits would commit (a throwaway index), -# so any edit, committed or not, makes the built binary stale. The build -# stamps it into the binary as og-tree-hash:; so the check -# reads it with grep and never runs a binary the host may not be able to. +# The source stamp hashes two git object IDs: rust/'s tree as it is on +# disk and the blob of assets/imprinted_regions.tsv. The latter is embedded +# at compile time, so an edit to it must also make a binary stale. The +# legacy tree-hash verb and og-tree-hash:; stamp keep their format. +# Checks read the stamp without executing a binary for another platform. set -euo pipefail # Steps 04, 05, 07, 09b, 21, 25, 32 and 33 (and the modules of 09b, 21 and 32: @@ -57,7 +57,7 @@ names() { } # tree_hash: exit 3 when this is not a git checkout, 1 when git fails. -tree_hash() { +rust_tree_hash() { local idx rc=0 git -C "$ROOT" rev-parse --verify -q HEAD:rust > /dev/null 2>&1 || return 3 if [ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all -- rust 2> /dev/null)" ]; then @@ -72,6 +72,22 @@ tree_hash() { [ "$rc" -eq 0 ] || return 1 } +# Hash the two source object IDs without writing to the checkout's object +# store. Both committed and uncommitted table edits change the stamp. +source_hash() { printf '%s\n%s\n' "$1" "$2" | git -C "$ROOT" hash-object --stdin; } +tree_hash() { + local rust asset + rust=$(rust_tree_hash) || return $? + asset=$(git -C "$ROOT" hash-object -- assets/imprinted_regions.tsv) || return 1 + source_hash "$rust" "$asset" +} +committed_hash() { + local rust asset + rust=$(git -C "$ROOT" rev-parse HEAD:rust) || return 1 + asset=$(git -C "$ROOT" rev-parse HEAD:assets/imprinted_regions.tsv) || return 1 + source_hash "$rust" "$asset" +} + # embedded FILE: the hash stamped into a binary, or "unknown". embedded() { local h @@ -176,7 +192,8 @@ build() { zigbuild() { "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target -e OG_TREE_HASH="$hash" \ - -v "${ROOT}/rust:/src:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ + -v "${ROOT}/rust:/src:ro" \ + -v "${ROOT}/assets/imprinted_regions.tsv:/assets/imprinted_regions.tsv:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ cargo zigbuild --release --locked -p og "$@" } zigbuild "${targets[@]}" || return @@ -250,7 +267,7 @@ install() { local tag rc=0 hash hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout; clone the repository to install og" >&2; return 1; } tag=$(git -C "$ROOT" describe --exact-match --tags --match 'v[0-9]*.[0-9]*.[0-9]*' HEAD 2> /dev/null || true) - if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [ "$hash" = "$(git -C "$ROOT" rev-parse HEAD:rust)" ]; then + if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [ "$hash" = "$(committed_hash)" ]; then echo "Release ${tag}: downloading its og binaries" fetch "$tag" || rc=$? [ "$rc" -ne 0 ] || return 0 diff --git a/tests/test_og_check.sh b/tests/test_og_check.sh index e4f2e6b43..0657ab3f3 100755 --- a/tests/test_og_check.sh +++ b/tests/test_og_check.sh @@ -27,7 +27,7 @@ expect() { # expect NAME RC WANT_RC OUTPUT PATTERN C="${T}/repo" mkdir -p "$C" -cp -R "${REPO}/scripts" "${REPO}/bin" "${REPO}/rust" "${REPO}/versions.env" "${REPO}/nextflow.config" "${REPO}/.gitignore" "$C/" +cp -R "${REPO}/scripts" "${REPO}/bin" "${REPO}/rust" "${REPO}/assets" "${REPO}/versions.env" "${REPO}/nextflow.config" "${REPO}/.gitignore" "$C/" rm -rf "${C}/bin/.og" "${C}/rust/target" g() { git -C "$C" -c user.name=t -c user.email=t@example.invalid "$@"; } g init -q && g add -A && g commit -qm base @@ -50,8 +50,11 @@ plant_mac() { # plant_mac HASH: the macOS pair, stamped with HASH done } +committed_hash() { + printf '%s\n%s\n' "$(g rev-parse HEAD:rust)" "$(g rev-parse HEAD:assets/imprinted_regions.tsv)" | g hash-object --stdin +} h0=$(bash "$OG" tree-hash) -[ "$h0" = "$(g rev-parse HEAD:rust)" ] && ok "tree-hash of a clean rust/ is HEAD:rust" || bad "tree-hash ${h0} != HEAD:rust" +[ "$h0" = "$(committed_hash)" ] && ok "source hash covers rust/ and the embedded table" || bad "source hash omits an input" out=$(bash "$OG" check); rc=$? expect "no binary: a note, exit 0" "$rc" 0 "$out" "NOTE: og is not built for x86_64 aarch64*setup.sh --og*" @@ -60,6 +63,14 @@ expect "no binary with --required: refused" "$rc" 1 "$out" "ERROR: og is not bui out=$(bash "$OG" built); [ "$out" = "not built" ] && ok "built without binaries: not built" || bad "built printed '${out}'" plant "$h0" +cp "${C}/assets/imprinted_regions.tsv" "${T}/imprinted.saved" +printf '\n# edited source\n' >> "${C}/assets/imprinted_regions.tsv" +ha=$(bash "$OG" tree-hash) +[ "$ha" != "$h0" ] && ok "an embedded table edit changes the source hash" || bad "embedded table edit did not change the hash" +out=$(bash "$OG" check); rc=$? +expect "an embedded table edit makes the binary stale" "$rc" 1 "$out" "ERROR: og is stale:*" +cp "${T}/imprinted.saved" "${C}/assets/imprinted_regions.tsv" +[ "$(bash "$OG" tree-hash)" = "$h0" ] && ok "restoring the table restores the hash" || bad "restored table has a different hash" out=$(bash "$OG" check); rc=$? expect "binaries of this rust/: OK" "$rc" 0 "$out" "OK: og is built for x86_64, aarch64*${h0}*" out=$(bash "$OG" built); [ "$out" = "$h0" ] && ok "built prints the one hash" || bad "built printed '${out}'" @@ -91,7 +102,7 @@ h1=$(bash "$OG" tree-hash) out=$(bash "$OG" check); rc=$? expect "uncommitted edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*${h0}*${h1}*run scripts/setup.sh --og" g add rust/og/src/main.rs && g commit -qm edit -[ "$(bash "$OG" tree-hash)" = "$h1" ] && [ "$h1" = "$(g rev-parse HEAD:rust)" ] \ +[ "$(bash "$OG" tree-hash)" = "$h1" ] && [ "$h1" = "$(committed_hash)" ] \ && ok "committing the edit keeps the hash the build would stamp" || bad "hash after commit differs" out=$(bash "$OG" check); rc=$? expect "committed edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*" @@ -213,6 +224,13 @@ else echo "SKIP release download cases: no python3 or curl" fi +# The builder must mount the compile-time table beside /src. The fake +# engine refuses its absence and stops before any cargo invocation. +printf '#!/usr/bin/env bash\nfor arg in "$@"; do\n case $arg in *"/assets/imprinted_regions.tsv:/assets/imprinted_regions.tsv:ro") echo EMBEDDED_TABLE_MOUNT_OK; exit 99 ;; esac\ndone\necho EMBEDDED_TABLE_MOUNT_MISSING\nexit 98\n' > "${T}/fake/embed-engine" +chmod 755 "${T}/fake/embed-engine" +out=$(PATH="${T}/fake:${PATH}" CONTAINER_ENGINE=embed-engine bash "$OG" build 2>&1); rc=$? +expect "the builder mounts its embedded table" "$rc" 99 "$out" "*EMBEDDED_TABLE_MOUNT_OK*" + # Outside git the hash cannot be checked: a note, never a silent OK. N="${T}/nogit" plant "$h1" From e6fe655b8ababf325feea5e37c08b135c458ad38 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:23:24 +0100 Subject: [PATCH 66/86] fix(docs): link the imprinting table through the public repository The built site cannot link outside its own root. The strict local build passes, and site-check reports 97 pages with zero errors. Bead: lic-41 --- docs/11-roh-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 07fc1e338..c9fe48202 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -85,7 +85,7 @@ In uniparental disomy (UPD) both copies of a chromosome come from one parent. Wh The reports flag a chromosome when its segments of 5 Mb or more (band parts left out) add up to at least 20 Mb **and** to more than half of the length of all such autosomal segments. Both numbers are our thresholds, not a published cut-off: 20 Mb on one chromosome is the long-standing sign of isodisomy (see the uniparental disomy note under [Important Notes](#important-notes)), and the share rule keeps out a child of related parents, whose runs spread over many chromosomes. The flag says "may": a run that long can also be chance in a child of related parents. -A flagged chromosome is looked up in [assets/imprinted_regions.tsv](../assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. +A flagged chromosome is looked up in [assets/imprinted_regions.tsv](https://github.com/GeiserX/OwnGenome/blob/main/assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. The flag lists the chromosome, run length, share, region names and citations. It names no syndrome. The existing `maternal_upd` and `paternal_upd` JSON keys stay as `"-"` for reader compatibility. The asset retains their reference facts. Those facts cannot be inferred from a region match in one genome. From 2bb5acc7d7bc1da22318852c398e1c210af963fd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:34:21 +0100 Subject: [PATCH 67/86] fix(archive): regenerate MD/NM when restoring a CRAM without stored tags Restore and CRAM_TO_BAM turned MD/NM generation off for every CRAM, so a CRAM written without store_md/store_nm (an older step 34, or sarek) lost the tags samtools drops where they match the reference. Turn generation off only when the CRAM's last writer stored both tags, and decode the CRAM the same way on both sides of the content check. The e2e case now restores a plain CRAM in bash and Nextflow and checks NM, and proves the module content checks fail on a changed CRAM-side checksum. --- CHANGELOG.md | 2 +- docs/34-cram-archive.md | 2 +- modules/local/cram_archive/main.nf | 14 +++++++++++--- scripts/34-cram-archive.sh | 27 ++++++++++++++++++++++----- tests/e2e/qc-3-cram-checksum.sh | 29 +++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3a75bfd..0ab34acd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. +- CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. An older or external CRAM written without stored MD/NM gets them regenerated on restore, as before. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/34-cram-archive.md b/docs/34-cram-archive.md index d4d74b062..c9fc9b1bb 100644 --- a/docs/34-cram-archive.md +++ b/docs/34-cram-archive.md @@ -36,7 +36,7 @@ samtools checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' --input-fmt-option The CRAM must pass `samtools quickcheck`, the two `flagstat` reports must match, and the content checksums must match. The checksum reads every record, including secondary and supplementary alignments. It covers read names, sequences, qualities, flags, positions, CIGARs, mate fields and all auxiliary tags. A changed base quality or tag can leave `flagstat` counts unchanged and still fail this check. A decoding error also fails the step. -The checksums use samtools' order-independent aggregation. The comparison ignores the filename header and sorts the report rows, so read and tag order do not affect it. samtools sanitizes format-level differences, such as an unmapped read's MAPQ and equivalent CIGARs. Explicit `MD` and `NM` tags are stored in the CRAM. They are not regenerated during the check or restore, so missing tags stay missing. These CRC-based content checks detect conversion faults. They are not a cryptographic identity proof. See the [samtools 1.24 checksum manual](https://www.htslib.org/doc/1.24/samtools-checksum.html). +The checksums use samtools' order-independent aggregation. The comparison ignores the filename header and sorts the report rows, so read and tag order do not affect it. samtools sanitizes format-level differences, such as an unmapped read's MAPQ and equivalent CIGARs. Explicit `MD` and `NM` tags are stored in the CRAM. For a CRAM this step wrote, they are not regenerated during the check or restore, so missing tags stay missing. An older or external CRAM, written without stored `MD`/`NM`, keeps only the tags that differ from the reference. Restoring it regenerates `MD` and `NM` from the reference, and the check decodes the CRAM the same way. These CRC-based content checks detect conversion faults. They are not a cryptographic identity proof. See the [samtools 1.24 checksum manual](https://www.htslib.org/doc/1.24/samtools-checksum.html). Only after all checks pass does the CRAM lose its `.part` name. A mismatch or a failed or empty checksum command removes the partial CRAM, keeps the BAM and its index, and exits non-zero. This also applies with `--delete-bam`. An older published CRAM stays in place if a new attempt fails. Every archive attempt writes a fresh CRAM from the current BAM before it can delete anything. The checksum adds a full read of each file to the archive and restore checks. diff --git a/modules/local/cram_archive/main.nf b/modules/local/cram_archive/main.nf index 069e98eaa..74912d13c 100644 --- a/modules/local/cram_archive/main.nf +++ b/modules/local/cram_archive/main.nf @@ -117,7 +117,15 @@ process CRAM_TO_BAM { script: """ - samtools view -@ ${task.cpus} -b --reference ${reference} --input-fmt-option decode_md=0 -o ${meta.id}_from_cram.bam ${cram} + # A CRAM whose last writer stored MD and NM (CRAM_ARCHIVE, step 34) decodes + # with generation off, so a tag the BAM lacked stays missing. Any other + # CRAM (an older step 34, sarek) holds only the MD/NM that differ from the + # reference and decodes with the samtools default, which regenerates them. + decode="" + if samtools view -H --no-PG ${cram} | awk '/^@PG/ { last = \$0 } END { exit !(last ~ /store_md=1/ && last ~ /store_nm=1/) }'; then + decode="--input-fmt-option decode_md=0" + fi + samtools view -@ ${task.cpus} -b --reference ${reference} \$decode -o ${meta.id}_from_cram.bam ${cram} samtools index -@ ${task.cpus} ${meta.id}_from_cram.bam samtools quickcheck -v ${meta.id}_from_cram.bam samtools flagstat -@ ${task.cpus} --input-fmt-option reference=${reference} ${cram} > cram.flagstat @@ -128,13 +136,13 @@ process CRAM_TO_BAM { fi # Match every record field and tag, independent of read/tag/group order. - # MD/NM are stored explicitly and never regenerated while checking. + # The CRAM decodes the way the BAM was written from it. # pipefail must propagate decoder failures through tail and sort. set -o pipefail checksum_rows() { samtools checksum -@ ${task.cpus} -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "\$@" | tail -n +2 | LC_ALL=C sort } - checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${cram} > cram.checksum + checksum_rows --input-fmt-option reference=${reference} \$decode ${cram} > cram.checksum checksum_rows ${meta.id}_from_cram.bam > bam.checksum if [ ! -s cram.checksum ] || [ ! -s bam.checksum ] || ! diff cram.checksum bam.checksum >&2; then echo "ERROR: the restored BAM content differs from ${cram}" >&2 diff --git a/scripts/34-cram-archive.sh b/scripts/34-cram-archive.sh index 0b50a0e3b..b0da7efa0 100755 --- a/scripts/34-cram-archive.sh +++ b/scripts/34-cram-archive.sh @@ -89,15 +89,28 @@ same_reads() { fi } +# How a CRAM is decoded. This step's encoder stores MD and NM explicitly, so +# its CRAMs decode with generation off and a tag the BAM lacked stays missing. +# A CRAM written without them (an older step 34, or one made elsewhere) holds +# only the MD/NM that differ from the reference: it decodes with samtools' +# default, which regenerates them. Both sides of a check decode the same way. +CRAM_DECODE=(--input-fmt-option decode_md=0) + +# md_stored CRAM_IN_CONTAINER: the last program that wrote the CRAM stored MD +# and NM (its @PG command line has store_md=1 and store_nm=1). +md_stored() { + run_in "$SAMTOOLS_IMAGE" samtools view -H --no-PG "$1" \ + | awk '/^@PG/ { last = $0 } END { exit !(last ~ /store_md=1/ && last ~ /store_nm=1/) }' +} + # The checksum reads every block, unlike flagstat. Use all record fields and # tags, including secondary/supplementary records, with order-independent # aggregation. Normalize the format-level repairs CRAM makes to unmapped -# records and CIGARs. Keep explicit MD/NM in the encoder and disable their -# generation on decode, so missing tags stay missing. Only the filename -# header is removed; sorting makes read-group row order immaterial too. +# records and CIGARs. Only the filename header is removed; sorting makes +# read-group row order immaterial too. checksum_rows() { local -a ref=() - case "$1" in *.cram) ref=(--input-fmt-option "reference=${REF_FASTA_C}" --input-fmt-option decode_md=0) ;; esac + case "$1" in *.cram) ref=(--input-fmt-option "reference=${REF_FASTA_C}" ${CRAM_DECODE[@]+"${CRAM_DECODE[@]}"}) ;; esac run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ samtools checksum -@ "$THREADS" -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' \ ${ref[@]+"${ref[@]}"} "$1" | tail -n +2 | LC_ALL=C sort @@ -126,8 +139,12 @@ if [ "$MODE" = --restore ]; then exit 1 } rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" + if ! md_stored "${C}.cram"; then + CRAM_DECODE=() + echo " The CRAM has no stored MD/NM tags: the BAM gets them regenerated from the reference." + fi run_in --cpus "$THREADS" --memory 4g "$SAMTOOLS_IMAGE" \ - samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" --input-fmt-option decode_md=0 \ + samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" ${CRAM_DECODE[@]+"${CRAM_DECODE[@]}"} \ -o "${C}.part.bam" "${C}.cram" || fail_restore "samtools could not write the BAM from the CRAM." run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ samtools index -@ "$THREADS" "${C}.part.bam" || fail_restore "samtools could not index the BAM." diff --git a/tests/e2e/qc-3-cram-checksum.sh b/tests/e2e/qc-3-cram-checksum.sh index acb4ce323..be74b9957 100755 --- a/tests/e2e/qc-3-cram-checksum.sh +++ b/tests/e2e/qc-3-cram-checksum.sh @@ -112,6 +112,20 @@ check "faithful restore writes BAM and index" test -s "${A}/${C}_sorted.bam" -a sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/original.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/original.checksum" sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/${C}_sorted.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/restored.checksum" check "round trip preserves read content and explicit tags" cmp -s "${CASE_TMP}/original.checksum" "${CASE_TMP}/restored.checksum" +# A CRAM written without store_md/store_nm (an older step 34, or one made +# elsewhere) drops MD and NM wherever they match the reference. Restoring it +# regenerates them, so read0's NM:i:0 is back in the BAM. +plain_cram() { + sam view -C -T reference/GRCh38_no_alt_analysis_set.fasta -o "$1" "${C}/aligned/original.bam" + sam index "$1" +} +nm_of_read0() { sam view "$1" | awk -F '\t' '$1 == "read0" { for (i = 12; i <= NF; i++) if ($i ~ /^NM:i:/) print $i }'; } +plain_cram "${C}/aligned/${C}_sorted.cram" +check "the plain CRAM stores no NM for read0" test -z "$(sam view -T reference/GRCh38_no_alt_analysis_set.fasta --input-fmt-option decode_md=0 "${C}/aligned/${C}_sorted.cram" | awk -F '\t' '$1 == "read0" && /NM:i:/')" +rm -f "${A}/${C}_sorted.bam" "${A}/${C}_sorted.bam.bai" +run_step 34-cram-archive.sh "$C" --restore +check_step_exit 34-cram-archive.sh +check_eq "restore of a plain CRAM regenerates NM" "$(nm_of_read0 "${C}/aligned/${C}_sorted.bam")" "NM:i:0" # Run the module's actual script blocks on the same synthetic records. Only # Nextflow interpolation and version metadata are omitted by this harness. # Literal Nextflow placeholders are substituted by sed, not by this shell. @@ -135,6 +149,10 @@ module_script() { # The function expands its arguments in the container shell. # shellcheck disable=SC2016 EMPTY_CHECKSUM='samtools() { if [ "$1" = checksum ]; then return 0; else command samtools "$@"; fi; }' +# A checksum that differs on the CRAM side only: the last report row's +# combined value is replaced. The module must fail on the diff alone. +# shellcheck disable=SC2016 +CHANGED_CHECKSUM='samtools() { if [ "$1" = checksum ]; then case "${@: -1}" in *.cram) command samtools "$@" | sed "\$s/[0-9a-f]\{8\}\$/00000000/" ;; *) command samtools "$@" ;; esac; else command samtools "$@"; fi; }' for process in CRAM_ARCHIVE CRAM_TO_BAM; do body=$(module_script "$process") rc=0 @@ -144,5 +162,16 @@ for process in CRAM_ARCHIVE CRAM_TO_BAM; do in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "${EMPTY_CHECKSUM} ${body}" || rc=$? check "${process}: empty checksum reports fail" test "$rc" -ne 0 + rc=0 + in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "${CHANGED_CHECKSUM} +${body}" > "${CASE_TMP}/changed-${process}.log" 2>&1 || rc=$? + check "${process}: a changed CRAM checksum fails" test "$rc" -ne 0 + check "${process}: the failure is the content check" has "content .*differs" "$(cat "${CASE_TMP}/changed-${process}.log")" done +# CRAM_TO_BAM on a CRAM without stored MD/NM regenerates them, like --restore. +plain_cram module_sorted.cram +rc=0 +in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "$(module_script CRAM_TO_BAM)" || rc=$? +check_eq "CRAM_TO_BAM: plain CRAM passes" "$rc" 0 +check_eq "CRAM_TO_BAM: plain CRAM regenerates NM" "$(nm_of_read0 module_from_cram.bam)" "NM:i:0" finish From beb57c97425e9bbcfd689e46fdd8135cff2ac3c4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:34:24 +0100 Subject: [PATCH 68/86] fix(recheck): count failed items, not failed lookups, in the last line An item also fails when the run's manifest lacks the value to compare, which no later lookup fixes, so the closing line counts looked-up items that failed and points at each item's own line for why. Bead: plat-26 --- CHANGELOG.md | 2 +- docs/recheck.md | 2 +- rust/crates/og-recheck/src/lib.rs | 10 ++++++---- rust/crates/og-recheck/src/tests.rs | 11 ++++++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 574c7d7a1..0fef8601b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many lookups failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) +- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many of the looked-up items failed and that the check is incomplete, or "Could not check: all N looked-up items failed" when none could be checked. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/recheck.md b/docs/recheck.md index 0d149ec73..bfc471eb2 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -51,7 +51,7 @@ Each item gets one class: A newer file or pin installed here stays `rerun-ready` when its lookup fails, because that comparison needs no network. The item then keeps its `error` too, and the command still exits 1. -The last line of the output sums up the check. When something is `rerun-ready` or `refresh-available`, it names the steps a rerun would redo. When a lookup failed, it says how many of how many failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. "Nothing here is newer than what the run used." comes only when every lookup answered and nothing is newer. +The last line of the output sums up the check. When something is `rerun-ready` or `refresh-available`, it names the steps a rerun would redo. When an item failed, because its lookup failed or the run's manifest lacks the value to compare, it says how many of the looked-up items failed and that the check is incomplete, or "Could not check: all N looked-up items failed" when none could be checked. Each item's own line says why. "Nothing here is newer than what the run used." comes only when every looked-up item was checked and nothing is newer. The 2-day margin for ClinVar is there because NCBI dates the file a day or two after the release in its `##fileDate`. A newer PGS file date means the catalog wrote the harmonised file again; it does not prove the weights changed. diff --git a/rust/crates/og-recheck/src/lib.rs b/rust/crates/og-recheck/src/lib.rs index b966f3159..aca5f5bd4 100644 --- a/rust/crates/og-recheck/src/lib.rs +++ b/rust/crates/og-recheck/src/lib.rs @@ -792,7 +792,9 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( )?; } // A failed lookup is never read as "nothing newer". An item that was - // looked up has an answer or an error; the others were never asked. + // looked up has an answer or an error; the others were never asked. An + // error is a failed lookup or a run value missing from the manifest, so + // the count is of items, and each item's line says which. let asked = c .items .iter() @@ -806,13 +808,13 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( } else if failed >= asked { writeln!( out, - "Could not check: all {asked} lookups failed (each line above says why). Check again later." + "Could not check: all {asked} looked-up items failed (each line above says why)." )?; } else { writeln!( out, - "{failed} of {asked} lookups failed: this check is incomplete, so it cannot say \ - nothing else is newer. Check again later." + "{failed} of {asked} looked-up items failed (each line above says why): this check is \ + incomplete, so it cannot say nothing else is newer." )?; } Ok(()) diff --git a/rust/crates/og-recheck/src/tests.rs b/rust/crates/og-recheck/src/tests.rs index fb1fd6bea..a19d8dd4f 100644 --- a/rust/crates/og-recheck/src/tests.rs +++ b/rust/crates/og-recheck/src/tests.rs @@ -320,6 +320,11 @@ fn what_cannot_be_compared_is_an_error() { let cv = by_item(&c)["clinvar_file_date"]; assert_eq!(cv.class, Class::Error); assert!(cv.error.as_deref().unwrap().contains("which ClinVar file")); + // A run value missing from the manifest is no "nothing newer" either, + // and is not called a failed lookup. + let out = printed(&c); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!(out.contains("1 of 7 looked-up items failed"), "{out}"); let g = genome( "nodate", Opt { @@ -1133,7 +1138,7 @@ fn a_failed_lookup_never_prints_nothing_newer_and_keeps_a_newer_file_here_rerun_ assert!(!out.contains("Nothing here is newer"), "{out}"); assert!(out.contains("the lookup failed: no answer"), "{out}"); assert!(out.contains("A rerun would redo steps 06,"), "{out}"); - assert!(out.contains("1 of 7 lookups failed"), "{out}"); + assert!(out.contains("1 of 7 looked-up items failed"), "{out}"); } #[test] @@ -1158,7 +1163,7 @@ fn when_every_lookup_is_refused_the_check_says_it_could_not_check() { let out = printed(&c); assert!(!out.contains("Nothing here is newer"), "{out}"); assert!( - out.contains("Could not check: all 7 lookups failed"), + out.contains("Could not check: all 7 looked-up items failed"), "{out}" ); // The command: every answer an error, exit 1, the same closing line. @@ -1182,7 +1187,7 @@ fn when_every_lookup_is_refused_the_check_says_it_could_not_check() { assert_eq!(rc, 1, "{out}"); assert!(!out.contains("Nothing here is newer"), "{out}"); assert!( - out.contains("Could not check: all 7 lookups failed"), + out.contains("Could not check: all 7 looked-up items failed"), "{out}" ); // Control: with every answer given, nothing newer is said. From def428aee61fb8900e710e4260790531d00147bc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:40:32 +0100 Subject: [PATCH 69/86] fix(live): clear reads links and labels when the panel closes Bead: pgp-iqon.11 --- CHANGELOG.md | 2 +- assets/live/map.js | 4 ++++ docs/viewer.md | 2 +- tests/test_live_viewer.py | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9382a20d1..fdf953ca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer; Esc closes it and returns focus to the map. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/assets/live/map.js b/assets/live/map.js index 0af553b75..43105cd81 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -1251,6 +1251,10 @@ if (reads.hidden) { return; } reads.hidden = true; readsBody.textContent = ''; // stop reads requests, and discard a result that can close again + readsPlace.textContent = ''; + readsWho.textContent = ''; + readsFull.removeAttribute('href'); + readsFull.hidden = true; readsFrame = null; readsId = null; readsLocus = ''; readsView = ''; readsPill.setAttribute('aria-expanded', 'false'); inview.hidden = !!failure; diff --git a/docs/viewer.md b/docs/viewer.md index 8b30532c8..b6e6d1f49 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -84,7 +84,7 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. -The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. +The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. Closing the panel clears its previous locus, title, and link to the full viewer. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. The live viewer allows framing only by the same origin: it sends `frame-ancestors 'self'` and `X-Frame-Options: SAMEORIGIN`. Live pages allow `frame-src 'self'`. Other routes keep `frame-ancestors 'none'` and `X-Frame-Options: DENY`. The standalone local viewer keeps its original policy, which blocks all framing. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index df50d0d61..288bd61b4 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -308,7 +308,9 @@ def loaded(): check("map: an opened finding offers reads", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:501-601'")) b.eval("window.closedResult=true;testMap.refresh('one')") check("map: closing a result removes its open reads panel and link", - b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads')")) + b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads') && " + "!document.querySelector('.gm-reads a[href]') && " + "!document.querySelector('.gm-reads').textContent.includes('Closed finding')")) # No finding in view: use the view itself, and no viewer flags: show the server's reason. b.eval("location.hash='chr1:10000-59000'") From 604fd42ec5c55be579c7a09ee8a94f207797b044 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:54:07 +0100 Subject: [PATCH 70/86] test(pgx): a split 1/2 site in a recessive gene must warn, not read as a carrier --- rust/crates/og-pgx/src/sections.rs | 4 ++-- .../og-pgx/tests/i411_contraindications.rs | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index f2e59a628..7ce151301 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -785,10 +785,10 @@ mod tests { #[test] fn genotypes_with_both_copies_changed() { - for gt in ["1/1", "1|1", "1/2", "2|1", "1"] { + for gt in ["1/1", "1|1", "1/2", "2|1", "1", "./1", "1/.", ".|1"] { assert!(both_copies(gt), "{gt}"); } - for gt in ["0/1", "1|0", "./1", "1/.", "0", "", "./."] { + for gt in ["0/1", "1|0", "0/.", "./0", "0", "", ".", "./."] { assert!(!both_copies(gt), "{gt}"); } } diff --git a/rust/crates/og-pgx/tests/i411_contraindications.rs b/rust/crates/og-pgx/tests/i411_contraindications.rs index 3d31046d3..208a15b8a 100644 --- a/rust/crates/og-pgx/tests/i411_contraindications.rs +++ b/rust/crates/og-pgx/tests/i411_contraindications.rs @@ -59,8 +59,8 @@ fn one_heterozygous_urea_cycle_hit_is_a_note() { "{gene}: {out:#?}" ); } - // A phased or partly missing genotype is still one changed copy. - for gt in ["0|1", "1|0", "./1"] { + // A phased genotype is still one changed copy. + for gt in ["0|1", "1|0"] { let out = lines(&[hit("ASS1", "40000000", "A", gt)]); assert!(warnings(&out).is_empty(), "{gt}: {out:#?}"); } @@ -76,6 +76,19 @@ fn both_copies_or_two_variants_in_a_urea_cycle_gene_warn() { "{gt}: {out:#?}" ); } + // Step 06 splits a 1/2 record into 1/. and ./1: the missing allele is + // the other ALT, so both copies are changed. + for gt in ["1/.", "./1"] { + let out = lines(&[hit("ASS1", "40000000", "A", gt)]); + assert_eq!( + warnings(&out), + [format!( + "WARNING: valproic acid -- ASS1 chr13:40000000 G>A, genotype {gt}, ClinVar Pathogenic, both copies changed in this recessive gene (the other copy is not the reference); ON THE MEDICATION LIST." + )], + "{gt}: {out:#?}" + ); + assert!(!out.iter().any(|l| l.contains("NOTE:")), "{gt}: {out:#?}"); + } // Two variants in trans, at two positions or as the two alleles of one // split multi-allelic site: one WARNING for each. for (pos2, alt2) in [("40000500", "A"), ("40000000", "C")] { From 83d39d68c91b88799f1c258bd02ce50b483d6a2e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:54:07 +0100 Subject: [PATCH 71/86] fix(pgx): count 1/. and ./1 as both copies changed in a recessive gene --- CHANGELOG.md | 2 +- docs/27-cpic-lookup.md | 2 +- rust/crates/og-pgx/src/sections.rs | 26 +++++++++++++++++--------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05209c768..25fe8aa67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed, or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. +- A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed (`1/.` and `./1`, the two alleles of a split `1/2` site, count as both copies), or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/27-cpic-lookup.md b/docs/27-cpic-lookup.md index 180009f50..c27c4dfb7 100644 --- a/docs/27-cpic-lookup.md +++ b/docs/27-cpic-lookup.md @@ -44,7 +44,7 @@ MEDICATIONS=/path/to/medications.txt ./scripts/27-cpic-lookup.sh your_name 5. With the pypgx summary it writes the comparison table and, for a gene PharmCAT reports as not called or ambiguous but pypgx called (CYP2D6 is the usual one), a warning in the recommendations naming every drug PharmCAT links to that gene. When pypgx's call is normal the line is a note instead. 6. With step 36's consensus table it writes a section "Calls From Other Tools": each gene passed to PharmCAT as an outside call, from which tool, and whether PharmCAT's report shows it as one (`callSource` `OUTSIDE`); and each gene held back, with the reason and what each caller said. For a held-back CYP2D6 it names the drugs CYP2D6 affects and says no guidance is given for them, and the pypgx-only warning of point 5 is not printed for it: one caller's call is what the consensus refused. Genes PharmCAT reports as outside calls carry `[outside call]` in the gene results. An outside call of CYP2D6, HLA-A or HLA-B in PharmCAT's report that the table does not pass on, with the same diplotype, comes from an earlier step 36 run that step 07 has not replaced: it gets no drug guidance, is listed as not called, and a warning says to rerun step 07. When the table is missing (a caller step ran again since) but PharmCAT's report has outside calls of those genes, the step passes the missing table, so none of them is confirmed. MT-RNR1's outside call comes from step 20's chrM calls through step 07, never from step 36: the table never lists it, so it is not checked against it and keeps its drug guidance. 7. With a medication list, each line is matched by its first word (a space, comma, semicolon, slash or plus ends a word, so `lopinavir/ritonavir` is matched as lopinavir; leading commas, semicolons, slashes and pluses are ignored, and a line of only those is skipped) against `assets/pgx_inhibitors.tsv`: the FDA's strong and moderate inhibitors of CYP2D6 and CYP2C19, its moderate CYP2C9 inhibitors (it lists no strong one) and its strong CYP3A inhibitors, one row each for CYP3A4 and CYP3A5, as the FDA's table of clinical CYP inhibitors classes them, each row with its source. A ritonavir-boosted combination the FDA classes as one (`lopinavir and ritonavir`) has a row under ritonavir and one under its partner. For CYP2D6 the step applies CPIC's rule from its opioid guideline: "for patients on strong CYP2D6 inhibitors, the CYP2D6 activity score is adjusted to 0 and the predicted phenotype is a poor metabolizer", and for a moderate inhibitor "the activity score is multiplied by 0.5 and then converted to the predicted phenotype" (CPIC's bands: 0 poor, under 1.25 intermediate, 1.25 to 2.25 normal, above 2.25 ultrarapid). The activity score is the one PharmCAT gives the called diplotype. With several inhibitors of one gene the strongest counts. The result is written as `Poor Metabolizer (phenoconverted by paroxetine)` beside the genotype phenotype, never in its place, with a `WARNING:` line in the section "Medications and Phenoconversion", so the summary carries it. Nothing is converted for a gene that is not called, ambiguous or unclassified, or for a moderate inhibitor when the report gives no activity score: the line says why. CPIC has no activity score for CYP2C19, and gives no rule that converts the phenotype of CYP2C9, CYP3A4 or CYP3A5, so an inhibitor of one of these genes gets a note and a `not computed: ...` value, with no converted phenotype. For a poor metabolizer of these genes (the most common CYP3A5 result) the note does not say the inhibitor can lower the activity further. A line that names no drug of the table is listed as "Not in the inhibitor table", never dropped. A list that names no drug converts nothing and leaves the phenotype table as it is without a list. An unreadable list or table is reported in that section; the rest of the report is written as usual. The FDA's table lists no strong or moderate CYP2B6 inhibitor, so CYP2B6 has no rows. -8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, `1/2`, a hemizygous `1`) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. +8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, a hemizygous `1`, or `1/.` and `./1`, which step 06 writes for the two alleles of a `1/2` site: the missing copy holds the other allele, not the reference) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. 9. A report that cannot be read, or that yields no gene, writes a "PARSING FAILED" report and the step exits 1. It never writes an all-clear report from a format it could not read. A field of the report with an unexpected type (a number where a name belongs, say) reads as missing. 10. The `Date:` line of the recommendations is today's date in UTC. The output files are written under a temporary name and renamed into place, so a step that stops half way leaves no half-written table. diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index 7ce151301..8bd5cd0b3 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -643,13 +643,14 @@ pub fn hit_genes(geneinfo: Option<&str>) -> BTreeSet { .collect() } -/// Every allele of the genotype is called and none is the reference: both -/// copies changed (`1/1`, `1|2`), or the one copy of a hemizygous call (`1`). -/// `0/1`, `1|0` and `./1` are one changed copy. +/// No allele of the genotype is the reference and at least one is a called +/// ALT: both copies changed (`1/1`, `1|2`), the one copy of a hemizygous +/// call (`1`), or one allele of a split multi-allelic site (`1/.`, `./1`). +/// Step 06 splits a `1/2` record into `1/.` and `./1`: the missing allele +/// is the other ALT, not the reference. `0/1`, `1|0` and `./.` are not. fn both_copies(genotype: &str) -> bool { - genotype - .split(['/', '|']) - .all(|a| !matches!(a.trim(), "" | "." | "0")) + let alleles = || genotype.split(['/', '|']).map(str::trim); + !alleles().any(|a| matches!(a, "" | "0")) && alleles().any(|a| a != ".") } /// The lines for each pathogenic or likely pathogenic hit in a gene of the @@ -707,9 +708,16 @@ pub fn contraindication_lines( ); match r.report_if { ReportIf::Any => lines.push(format!(" WARNING: {head}{on}.")), - ReportIf::Biallelic if both_copies(&h.genotype) => lines.push(format!( - " WARNING: {head}, both copies changed in this recessive gene{on}." - )), + ReportIf::Biallelic if both_copies(&h.genotype) => { + let why = if h.genotype.split(['/', '|']).any(|a| a.trim() == ".") { + " (the other copy is not the reference)" + } else { + "" + }; + lines.push(format!( + " WARNING: {head}, both copies changed in this recessive gene{why}{on}." + )); + } ReportIf::Biallelic if n >= 2 => lines.push(format!( " WARNING: {head}, {n} variants in {gene}, a recessive gene (phase not known){on}." )), From bdea2e905712d8636de5b02c5228ad9da32d8c41 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:10:09 +0100 Subject: [PATCH 72/86] test(setup): require clinical installs to resume in low space and report a skip --- tests/fake-docker/setup-clinical-default.sh | 40 +++++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh index 83a674b03..e83b145d9 100644 --- a/tests/fake-docker/setup-clinical-default.sh +++ b/tests/fake-docker/setup-clinical-default.sh @@ -28,11 +28,7 @@ output_has plan '^ 13 VEP +runs$' output_has plan '^ 23 Clinical filter +runs$' output_has plan '^ 17 CPSR +runs$' -# A second install does no network work, even with too little space. -cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" -run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" -cmp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" || fail 'installed clinical data downloaded again' - +# A fake df that reports 40 GiB free. mkdir -p "${CASE_WORK}/low-bin" cat > "${CASE_WORK}/low-bin/df" <<'DF' #!/usr/bin/env bash @@ -40,19 +36,49 @@ echo 'Filesystem 1024-blocks Used Available Capacity Mounted on' echo 'fakefs 41943040 0 41943040 0% /' DF chmod +x "${CASE_WORK}/low-bin/df" + +# A second install does no network work, even with too little space. +cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" +cmp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" || fail 'installed clinical data downloaded again' + +# A skipped item makes --clinical-data exit 1; the other items still install. G="${CASE_WORK}/low" -PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 0 low "${SCRIPTS}/setup.sh" --clinical-data "$G" +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 1 low "${SCRIPTS}/setup.sh" --clinical-data "$G" output_has low '56 GB.*5 GB margin' output_has low '52 GB.*5 GB margin' [ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'low-space VEP installed' [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'small bundle skipped with 40 GB free' +# A partial download counts as space already taken: 40 GiB free plus a +# 22 GiB .part covers VEP 116's 56 GB peak and margin, so it resumes. +G="${CASE_WORK}/resume" +mkdir -p "${G}/vep_cache" +dd if=/dev/null of="${G}/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz.part" bs=1048576 seek=22528 2>/dev/null +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 1 resume "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_lacks resume 'Skipping VEP 116' +[ -s "${G}/vep_cache/homo_sapiens/116_GRCh38/info.txt" ] || fail 'partial VEP 116 download not resumed with low space' +[ ! -e "${G}/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz.part" ] || fail 'resumed VEP 116 left its .part' + +# An install stopped during extraction keeps its checked archive: the next +# run unpacks it without downloading again and removes the old staging tree. +G="${CASE_WORK}/extract" +mkdir -p "${G}/vep_cache/.extract.old" "${G}/pcgr_data/.extract.old" +cp "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_116_GRCh38.tar.gz" "${G}/vep_cache/" +cp "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" "${G}/pcgr_data/" +FAKE_DOWNLOAD_FAIL='vep_116|pcgr_ref_data' run_expect 0 extract "${SCRIPTS}/setup.sh" --clinical-data "$G" +[ -s "${G}/vep_cache/homo_sapiens/116_GRCh38/info.txt" ] || fail 'checked VEP 116 archive downloaded again' +[ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'checked PCGR archive downloaded again' +[ ! -e "${G}/vep_cache/.extract.old" ] || fail 'stale VEP staging tree kept' +[ ! -e "${G}/pcgr_data/.extract.old" ] || fail 'stale PCGR staging tree kept' + # The digest belongs to the original fixture. Serve a different valid tarball. printf 'corrupt\n' >> "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" G="${CASE_WORK}/corrupt" -run_expect 0 corrupt "${SCRIPTS}/setup.sh" --clinical-data "$G" +run_expect 1 corrupt "${SCRIPTS}/setup.sh" --clinical-data "$G" output_has corrupt 'sha256 checksum.*expected' output_has corrupt '\[WARN\].*PCGR' [ ! -e "${G}/pcgr_data/20260620" ] || fail 'corrupt bundle published' [ ! -e "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz.part" ] || fail 'bad bundle kept' [ -s "${G}/vep_cache/homo_sapiens/115_GRCh38/info.txt" ] || fail 'bundle failure stopped remaining install' + From 91eb7722fa8a919bc3eefbbcb7bbef01e1d35634 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:15:25 +0100 Subject: [PATCH 73/86] fix(setup): let an interrupted clinical download resume in tight space and report a skipped item --- CHANGELOG.md | 2 +- docs/00-reference-setup.md | 5 +++-- docs/hardware-requirements.md | 2 +- docs/troubleshooting.md | 3 ++- scripts/lib/common.sh | 27 +++++++++++++++++++++------ scripts/setup.sh | 21 ++++++++++++++------- 6 files changed, 42 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f78b639b2..da7194bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin and warns without stopping the remaining installs. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, counts a partial download already on disk, and warns without stopping the remaining installs; `--clinical-data` exits 1 when one is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index b8f108743..b00f2c3b9 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -6,8 +6,8 @@ One-time downloads required before running the pipeline. Each heading below give A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. -Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 14 GB, and VEP 115 needs 52 GB. -Each check adds a 5 GB margin. An item that does not fit is skipped with a warning. Setup continues. +Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 15 GB, and VEP 115 needs 52 GB. +Each check adds a 5 GB margin and counts a partial download already on disk. An item that does not fit is skipped with a warning. Setup continues. ```bash ./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads @@ -15,6 +15,7 @@ Each check adds a 5 GB margin. An item that does not fit is skipped with a warni ``` An interrupted transfer resumes. Existing caches and bundles are kept. +`--clinical-data` exits 1 when any of the three is still missing after it runs. Without these data, `run-all.sh` skips the affected steps and records the reason. A bare `nextflow run` still needs its tools and data parameters. Setup does not change that selection. diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 0077edce2..a895b392a 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -57,7 +57,7 @@ One row per download, in GB as `wget` and `du -h` count them (1 GB = 2^30 bytes) | **Total, default run** | **~73-78 GB** | **~98-103 GB** | | | With `--no-clinical-data` (first three rows + Docker images) | ~16-21 GB | ~33-38 GB | | -While a VEP cache installs, the tarball and the unpacked tree are on disk together until the tarball is deleted (`install_vep_cache` in `scripts/lib/common.sh`): 56 GB peak for release 116 and 52 GB for release 115. The bundle needs 14 GB peak. Setup adds a 5 GB margin to each check and warns when it skips an item. The three clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. To retry only these downloads, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"`. +While a VEP cache installs, the tarball and the unpacked tree are on disk together until the tarball is deleted (`install_vep_cache` in `scripts/lib/common.sh`): 56 GB peak for release 116 and 52 GB for release 115. The bundle needs 15 GB peak. Setup adds a 5 GB margin to each check and warns when it skips an item. The three clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. To retry only these downloads, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"`. **Optional** (only for the step named): diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 120af6ba0..e052fb407 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -739,7 +739,8 @@ ExpansionHunter \ Setup installs VEP 116, the PCGR/CPSR bundle and VEP 115 by default. If one is missing, read the warning above `Setup complete!`. -The disk checks require 56 GB, 14 GB and 52 GB peak respectively, each plus a 5 GB margin. +The disk checks require 56 GB, 15 GB and 52 GB peak respectively, each plus a 5 GB margin. +A partial download on disk counts toward that peak. A failed download also leaves a warning. Other installs continue. Free space, then retry only the clinical data: diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 2088b40a6..923f082dc 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -642,8 +642,11 @@ install_vep_cache() { return 1 fi mkdir -p "${dir}/homo_sapiens" - fetch "$url" "$tarball" sum "$(dirname "$url")/CHECKSUMS" || return 1 + # fetch publishes the tarball only after its check, so one already here is + # whole: an install stopped during extraction unpacks it again. + [ -f "$tarball" ] || fetch "$url" "$tarball" sum "$(dirname "$url")/CHECKSUMS" || return 1 echo " Extracting $(basename "$tarball")..." + rm -rf "${dir}"/.extract.* tmp=$(mktemp -d "${dir}/.extract.XXXXXX") if ! tar xzf "$tarball" -C "$tmp" || [ ! -f "${tmp}/homo_sapiens/${rel}_GRCh38/info.txt" ]; then rm -rf "$tmp" @@ -657,18 +660,28 @@ install_vep_cache() { rm -rf "$tmp" "$tarball" } -# clinical_disk_ok DIR PEAK_GB LABEL: GB as du counts them (2^30 bytes), -# plus a 5 GB margin. +# clinical_disk_ok DIR PEAK_GB LABEL [ARCHIVE]: GB as du counts them +# (2^30 bytes), plus a 5 GB margin. # Use the destination's filesystem, creating the directory when it is new. +# The bytes of ARCHIVE, or of its interrupted ARCHIVE.part, are already part +# of the peak, so a resumed download needs only the rest. # An unreadable df is a failed preflight, never permission to fill the disk. clinical_disk_ok() { - local dir=$1 peak=$2 label=$3 free + local dir=$1 peak=$2 label=$3 archive=${4:-} free have=0 f mkdir -p "$dir" || return 1 free=$(df -Pk "$dir" 2>/dev/null | awk 'NR == 2 {print $4}') || free="" case "$free" in ''|*[!0-9]*) echo "[WARN] Cannot read free space for ${label}; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; return 1 ;; esac - if [ "$free" -lt "$(( (peak + 5) * 1048576 ))" ]; then + if [ -n "$archive" ]; then + for f in "$archive" "${archive}.part"; do + if [ -f "$f" ]; then + have=$(( $(wc -c < "$f") / 1024 )) + break + fi + done + fi + if [ "$(( free + have ))" -lt "$(( (peak + 5) * 1048576 ))" ]; then echo "[WARN] Skipping ${label}: needs ${peak} GB peak plus a 5 GB margin; $(awk -v k="$free" 'BEGIN {printf "%.1f", k / 1048576}') GB free." echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" return 1 @@ -691,7 +704,9 @@ install_pcgr_bundle() { url="https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" tarball="${dir}/${url##*/}" mkdir -p "$dir" || return 1 - fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 + # fetch publishes the archive only after its check, so one already here is whole. + [ -f "$tarball" ] || fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 + rm -rf "${dir}"/.extract.* tmp=$(mktemp -d "${dir}/.extract.XXXXXX") || return 1 if ! tar xzf "$tarball" -C "$tmp" || [ ! -d "${tmp}/data/grch38" ]; then rm -rf "$tmp" diff --git a/scripts/setup.sh b/scripts/setup.sh index 272808b4f..20d0fb79e 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -2082,7 +2082,7 @@ install_vep_cache_opt_in() { echo "[OK] VEP ${VEP_CACHE_RELEASE} cache already present: ${dir}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38" return 0 fi - clinical_disk_ok "$dir" 56 "VEP ${VEP_CACHE_RELEASE} cache" || return 1 + clinical_disk_ok "$dir" 56 "VEP ${VEP_CACHE_RELEASE} cache" "${dir}/$(basename "$(vep_cache_url "$VEP_CACHE_RELEASE")")" || return 1 echo "Installing the VEP ${VEP_CACHE_RELEASE} cache: ~26 GB download, ~30 GB unpacked, both on disk until it ends." if ! install_vep_cache "$dir" "$VEP_CACHE_RELEASE"; then echo "[WARN] Could not install the VEP ${VEP_CACHE_RELEASE} cache. Run this command again: the download resumes." @@ -2092,30 +2092,37 @@ install_vep_cache_opt_in() { } # Each missing item gets its own disk check. A skipped or failed item must -# not prevent the smaller bundle or the other cache from installing. +# not prevent the smaller bundle or the other cache from installing; it makes +# the return status 1. install_clinical_data() { - local dir="${GENOME_DIR}/vep_cache" rel peak + local dir="${GENOME_DIR}/vep_cache" rel peak missing=0 echo "Clinical data: about 57 GB to download, 65 GB unpacked; allow 1 to 3 hours." for rel in "$VEP_CACHE_RELEASE" bundle "$PCGR_VEP_CACHE_RELEASE"; do if [ "$rel" = bundle ]; then [ ! -d "${GENOME_DIR}/pcgr_data/${PCGR_DATA_BUNDLE}/data" ] || { echo "[OK] PCGR/CPSR bundle already present."; continue; } - if clinical_disk_ok "${GENOME_DIR}/pcgr_data" 14 'PCGR/CPSR bundle'; then - install_pcgr_bundle "${GENOME_DIR}/pcgr_data" || echo "[WARN] Could not install PCGR/CPSR. Step 17 needs it; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" + if clinical_disk_ok "${GENOME_DIR}/pcgr_data" 15 'PCGR/CPSR bundle' "${GENOME_DIR}/pcgr_data/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz"; then + install_pcgr_bundle "${GENOME_DIR}/pcgr_data" || { missing=1; echo "[WARN] Could not install PCGR/CPSR. Step 17 needs it; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; } + else + missing=1 fi else [ ! -f "${dir}/homo_sapiens/${rel}_GRCh38/info.txt" ] || { echo "[OK] VEP ${rel} cache already present."; continue; } peak=56 [ "$rel" != "$PCGR_VEP_CACHE_RELEASE" ] || peak=52 - if clinical_disk_ok "$dir" "$peak" "VEP ${rel} cache"; then + if clinical_disk_ok "$dir" "$peak" "VEP ${rel} cache" "${dir}/$(basename "$(vep_cache_url "$rel")")"; then if ! install_vep_cache "$dir" "$rel"; then + missing=1 echo "[WARN] Could not install VEP ${rel}. Run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\" (interrupted downloads resume)." if [ "$rel" = "$VEP_CACHE_RELEASE" ]; then echo " To retry only this cache: ./scripts/setup.sh --vep-cache \"${GENOME_DIR}\"" fi fi + else + missing=1 fi fi done + return "$missing" } case "$OPT_IN" in @@ -2510,7 +2517,7 @@ install_annotsv_data || true echo "" echo "=== Phase 6: Clinical reference data ===" if $CLINICAL_DATA; then - install_clinical_data + install_clinical_data || true else echo "[SKIP] Clinical data (--no-clinical-data): steps 13, 30, 23, 31 and 17 need the caches or bundle." echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" From b4be807a36c3c0d0201275b6fa574ee0e7664d3d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:15:50 +0100 Subject: [PATCH 74/86] fix(live): preserve reads history and locus across map navigation Replace iframe fragment navigation, keep the requested locus on resize, and add only one camera history entry from the Reads gauge. Bead: pgp-iqon.11 --- CHANGELOG.md | 2 +- assets/live/map.js | 10 ++++++---- assets/viewer/viewer.js | 9 ++++++++- docs/viewer.md | 2 ++ tests/test_live_viewer.py | 30 ++++++++++++++++++++++++++++-- 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdf953ca5..5de2d8186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/assets/live/map.js b/assets/live/map.js index 43105cd81..86f0c83df 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -1244,7 +1244,9 @@ readsFull.href = readsBase() + '#' + locus; if (readsFrame) { // Changing only the fragment lets viewer.js search without reloading igv.js. - readsFrame.src = readsFull.href; + if (readsFrame.parentNode && readsFrame.contentWindow) { + readsFrame.contentWindow.location.replace(readsFull.href); + } else { readsFrame.src = readsFull.href; } } } function closeReads() { @@ -1693,14 +1695,14 @@ c = f ? (f.s + f.e) / 2 : M.chroms[ci].len / 2; } if (lv === 'chromosome') { openChrom(ci, undefined, undefined, true); return; } - openChrom(ci, c, (lv === 'region' ? REGION_STOP : GENE_STOP) * openW(), true); + openChrom(ci, c, lv === 'reads' ? 2000 : (lv === 'region' ? REGION_STOP : GENE_STOP) * openW(), true); } stopC.addEventListener('click', function () { stopTo('chromosome'); }); stopR.addEventListener('click', function () { stopTo('region'); }); stopN.addEventListener('click', function () { stopTo('gene'); }); stopReads.addEventListener('click', function () { - stopTo('gene'); - openChrom(tgt.ci, tgt.c, 2000, true); + if (!M || failure) { return; } + stopTo('reads'); // The target is already at reads level; finish the camera motion before selecting its nearest row. cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; openReads(); diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index aadfcf506..558ea23cd 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -58,11 +58,18 @@ // igv.js otherwise fetches its genome list from igv.org at start. loadDefaultGenomes: false, }); - window.addEventListener("hashchange", function () { + function followLocus() { const locus = locusFromHash(); if (locus) { browser.search(locus); } + } + window.addEventListener("hashchange", followLocus); + // igv.js keeps bases per pixel on resize. Keep the requested absolute locus instead. + let resizeTimer; + window.addEventListener("resize", function () { + window.clearTimeout(resizeTimer); + resizeTimer = window.setTimeout(followLocus, 100); }); root.dataset.pgpTracks = String(cfg.tracks.length); root.dataset.pgpStatus = "ready"; diff --git a/docs/viewer.md b/docs/viewer.md index b6e6d1f49..09b8e3680 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -84,6 +84,8 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. +Back follows the map's history. Moving the map creates no extra history entry for the iframe, and the Reads gauge adds one camera view. Resizing the viewer keeps its requested locus. + The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. Closing the panel clears its previous locus, title, and link to the full viewer. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. The live viewer allows framing only by the same origin: it sends `frame-ancestors 'self'` and `X-Frame-Options: SAMEORIGIN`. Live pages allow `frame-src 'self'`. Other routes keep `frame-ancestors 'none'` and `X-Frame-Options: DENY`. The standalone local viewer keeps its original policy, which blocks all framing. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 288bd61b4..666a8b9f0 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -268,9 +268,12 @@ def loaded(): check("map: a closed result has no card or reads link", b.eval("!document.querySelector('.gm-marks [data-id=closed]') && !document.querySelector('.gm-card-reads[data-id=closed]')")) # A moved view must update the existing frame, with no page reload. - b.eval("window.savedFrame=document.querySelector('.gm-reads iframe');location.hash='chr1:1-49001'") + b.eval("window.savedFrame=document.querySelector('.gm-reads iframe');window.savedViewerDocument=savedFrame.contentDocument;" + "window.savedHistoryLength=history.length;location.hash='chr1:1-49001'") check("map: moving updates the same iframe to the map's locus", b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) + check("map: a camera move keeps the viewer document and adds no iframe history entry", + b.eval("savedViewerDocument===savedFrame.contentDocument && history.length===savedHistoryLength+1")) check("viewer: alignments remain visible at 49 kb", b.wait("(function(){var h=savedFrame.contentDocument.getElementById('igv');" "var v=h && h.shadowRoot && h.shadowRoot.querySelector('.igv-viewport[data-track-type=alignment]');" @@ -279,11 +282,31 @@ def loaded(): b.eval("JSON.stringify(Array.from(savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelectorAll('.igv-viewport')).map(function(v){" "var n=v.querySelector('.igv-zoom-in-notice-container');return {type:v.dataset.trackType," "notice:n && n.style.display,canvas:v.querySelectorAll('canvas').length,text:v.textContent.slice(0,180)};}))")) + def viewer_locus(): + return b.eval("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value") + + def resized(): + b.call("Runtime.evaluate", {"expression": "new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))", + "awaitPromise": True}, b.session) + + expected_locus = viewer_locus() + js("savedFrame.style.width='50%'") + resized() + check("viewer: resizing keeps the full requested locus", + b.wait("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value===" + + json.dumps(expected_locus)), viewer_locus()) + js("savedFrame.style.width=''") + resized() + b.wait("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value===" + + json.dumps(expected_locus)) if os.environ.get("MAP_SCREENS"): os.makedirs(os.environ["MAP_SCREENS"], exist_ok=True) for width, height in ((1440, 900), (390, 844)): b.call("Emulation.setDeviceMetricsOverride", {"width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, b.session) + resized() + b.wait("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value===" + + json.dumps(expected_locus)) b.eval("document.querySelector('.gm-reads').scrollIntoView({block:'center'})") png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] with open(os.path.join(os.environ["MAP_SCREENS"], f"map-reads-{width}.png"), "wb") as out: @@ -296,9 +319,12 @@ def loaded(): "document.querySelector('.gmap').dataset.view==='chr1:1-49001'")) b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown',{key:'r',bubbles:true}))") check("map: R reopens the panel", loaded()) - b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('[data-stop=reads]').click()") + b.eval("document.querySelector('.gm-reads-close').click();window.gaugeHistoryLength=history.length;" + "document.querySelector('[data-stop=reads]').click()") check("map: Reads gauge opens a 2 kb view and the panel", b.wait("!document.querySelector('.gm-reads').hidden && " "document.querySelector('.gmap').dataset.level==='reads'")) + check("map: the Reads gauge adds one camera history entry", + b.eval("history.length===gaugeHistoryLength+1")) js("document.querySelector('.gm-reads-close').click();location.hash='chr1:1-2000'") b.wait("document.querySelector('.gmap').dataset.view==='chr1:1-2000'") js("document.querySelector('.gm-marks [data-id=open]').focus();document.querySelector('.gm-card-reads').click()") From 1c83b9953982a93f6dac3f4fb164b10b313ece26 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:17:51 +0100 Subject: [PATCH 75/86] test(live): preserve the targeted reload control guards Keep the Reads navigation guard distinct from the two drawing guards the existing failed-reload control replaces. Bead: pgp-iqon.11 --- assets/live/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/live/map.js b/assets/live/map.js index 86f0c83df..990d02b28 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -1701,7 +1701,7 @@ stopR.addEventListener('click', function () { stopTo('region'); }); stopN.addEventListener('click', function () { stopTo('gene'); }); stopReads.addEventListener('click', function () { - if (!M || failure) { return; } + if (failure || !M) { return; } stopTo('reads'); // The target is already at reads level; finish the camera motion before selecting its nearest row. cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; From 4a52aabac50fb6e3c0325ab73e3b5ffbf0c1f8bf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:15:35 +0100 Subject: [PATCH 76/86] fix(ancestry): list the four planned verbs in og ancestry-fine --help The dispatcher already routes freqs, project, paint and simulate, but the usage line named only the six ported verbs. A test now checks that the help names every verb. --- rust/crates/og-ancestry/src/lib.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 743ea611a..633066b3d 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -551,8 +551,7 @@ fn is_negative_number(s: &str) -> bool { /// A verb's body: the parsed options and stdout. type Run = fn(&Args, &mut dyn io::Write) -> Result<()>; -const USAGE: &str = - "usage: og ancestry-fine {panel-map,check-map,order,sample-bed,chroms,report} ...\n"; +const USAGE: &str = "usage: og ancestry-fine {panel-map,check-map,order,sample-bed,chroms,report,freqs,project,paint,simulate} ...\n"; /// `og ancestry-fine [args]`: the exit code. Output goes to `out`, /// messages to `err`. @@ -750,4 +749,24 @@ mod tests { assert_eq!(rc, 0); assert!(out.starts_with("usage: og ancestry-fine report")); } + + #[test] + fn help_names_every_verb() { + let (rc, out, _) = testutil::run(&["--help"]); + assert_eq!(rc, 0); + for verb in [ + "panel-map", + "check-map", + "order", + "sample-bed", + "chroms", + "report", + "freqs", + "project", + "paint", + "simulate", + ] { + assert!(out.contains(verb), "{verb} missing from {out}"); + } + } } From ba3105dc277d49825bc3c173f8d6600f3c0fe233 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:15:35 +0100 Subject: [PATCH 77/86] build(live): keep og-live's results module visible to git The root .gitignore drops every results/ folder (Nextflow run output), so a new file under rust/crates/og-live/src/results/ was silently ignored. A crate-level .gitignore re-includes that one source folder. --- rust/crates/og-live/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 rust/crates/og-live/.gitignore diff --git a/rust/crates/og-live/.gitignore b/rust/crates/og-live/.gitignore new file mode 100644 index 000000000..73b89169b --- /dev/null +++ b/rust/crates/og-live/.gitignore @@ -0,0 +1,3 @@ +# The repo root ignores results/ (Nextflow run output); this crate keeps a +# source module of that name. +!/src/results/ From eade22a68121042f80862696fb5fa7aeb59a89d6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:16:36 +0100 Subject: [PATCH 78/86] test(setup): catch stale extraction space and unverified retained bundles --- tests/fake-docker/setup-clinical-default.sh | 41 ++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh index e83b145d9..4c6b49e46 100644 --- a/tests/fake-docker/setup-clinical-default.sh +++ b/tests/fake-docker/setup-clinical-default.sh @@ -50,6 +50,16 @@ output_has low '52 GB.*5 GB margin' [ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'low-space VEP installed' [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'small bundle skipped with 40 GB free' +# Model an allocated 22 GiB partial download without writing 22 GiB in CI. +cat > "${CASE_WORK}/low-bin/du" <<'DU' +#!/usr/bin/env bash +case "$2" in + */resume/vep_cache/*.part) printf '23068672\t%s\n' "$2" ;; + *) /usr/bin/du "$@" ;; +esac +DU +chmod +x "${CASE_WORK}/low-bin/du" + # A partial download counts as space already taken: 40 GiB free plus a # 22 GiB .part covers VEP 116's 56 GB peak and margin, so it resumes. G="${CASE_WORK}/resume" @@ -66,12 +76,41 @@ G="${CASE_WORK}/extract" mkdir -p "${G}/vep_cache/.extract.old" "${G}/pcgr_data/.extract.old" cp "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_116_GRCh38.tar.gz" "${G}/vep_cache/" cp "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" "${G}/pcgr_data/" -FAKE_DOWNLOAD_FAIL='vep_116|pcgr_ref_data' run_expect 0 extract "${SCRIPTS}/setup.sh" --clinical-data "$G" +# Model space held by each stale tree. Only cleanup can make its preflight pass. +mkdir -p "${CASE_WORK}/staging-bin" +cat > "${CASE_WORK}/staging-bin/df" <<'DF' +#!/usr/bin/env bash +echo 'Filesystem 1024-blocks Used Available Capacity Mounted on' +if [ -d "${@: -1}/.extract.old" ]; then + echo 'fakefs 2147483648 2141192192 6291456 99% /' +else + echo 'fakefs 2147483648 0 2147483648 0% /' +fi +DF +chmod +x "${CASE_WORK}/staging-bin/df" +PATH="${CASE_WORK}/staging-bin:${PATH}" FAKE_DOWNLOAD_FAIL='vep_116|pcgr_ref_data' run_expect 0 extract "${SCRIPTS}/setup.sh" --clinical-data "$G" [ -s "${G}/vep_cache/homo_sapiens/116_GRCh38/info.txt" ] || fail 'checked VEP 116 archive downloaded again' [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'checked PCGR archive downloaded again' [ ! -e "${G}/vep_cache/.extract.old" ] || fail 'stale VEP staging tree kept' [ ! -e "${G}/pcgr_data/.extract.old" ] || fail 'stale PCGR staging tree kept' +# A retained archive with a valid layout but wrong data must be replaced. +G="${CASE_WORK}/retained-bad" +mkdir -p "${G}/pcgr_data" "${CASE_WORK}/altered/data/grch38" +printf 'modified reference\n' > "${CASE_WORK}/altered/data/grch38/README" +tar -czf "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/altered" data +run_expect 0 retained_bad "${SCRIPTS}/setup.sh" --clinical-data "$G" +cmp "${G}/pcgr_data/20260620/data/grch38/README" "${CASE_WORK}/bundle/data/grch38/README" || fail 'unverified retained PCGR archive published' +[ ! -e "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz" ] || fail 'replaced archive kept' + +# Sparse holes occupy no disk: they must not grant space for an installation. +G="${CASE_WORK}/sparse" +mkdir -p "${G}/vep_cache" +dd if=/dev/null of="${G}/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz.part" bs=1048576 seek=22528 2>/dev/null +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 1 sparse "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_has sparse 'Skipping VEP 116' +[ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'sparse holes counted as allocated space' + # The digest belongs to the original fixture. Serve a different valid tarball. printf 'corrupt\n' >> "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" G="${CASE_WORK}/corrupt" From 690e8ae13c117645a1c4579a6eb179c6cac6df5d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:20:16 +0100 Subject: [PATCH 79/86] fix(setup): reclaim staging space and verify retained clinical archives --- CHANGELOG.md | 2 +- docs/00-reference-setup.md | 3 ++- docs/troubleshooting.md | 3 ++- scripts/lib/common.sh | 19 +++++++++++++++---- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da7194bc5..38f38ef96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, counts a partial download already on disk, and warns without stopping the remaining installs; `--clinical-data` exits 1 when one is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, credits the allocated space of a partial download, and warns without stopping the remaining installs. Stale extraction files are removed before the disk check. Retained PCGR archives are checked against the pinned digest before reuse. `--clinical-data` exits 1 when one item is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index b00f2c3b9..fa2236f7d 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -7,7 +7,8 @@ One-time downloads required before running the pipeline. Each heading below give A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 15 GB, and VEP 115 needs 52 GB. -Each check adds a 5 GB margin and counts a partial download already on disk. An item that does not fit is skipped with a warning. Setup continues. +Each check adds a 5 GB margin and credits the disk space occupied by a partial download. +Stale extraction files are removed before the check. Retained PCGR archives are checked against the pinned SHA-256 before reuse. An item that does not fit is skipped with a warning. Setup continues. ```bash ./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e052fb407..b75d734d2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -740,7 +740,8 @@ ExpansionHunter \ Setup installs VEP 116, the PCGR/CPSR bundle and VEP 115 by default. If one is missing, read the warning above `Setup complete!`. The disk checks require 56 GB, 15 GB and 52 GB peak respectively, each plus a 5 GB margin. -A partial download on disk counts toward that peak. +The disk space occupied by a partial download counts toward that peak. Sparse file holes do not count. +Setup removes stale extraction files before checking space and checks a retained PCGR archive against its pinned SHA-256 before reuse. A failed download also leaves a warning. Other installs continue. Free space, then retry only the clinical data: diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 923f082dc..17374e740 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -663,12 +663,15 @@ install_vep_cache() { # clinical_disk_ok DIR PEAK_GB LABEL [ARCHIVE]: GB as du counts them # (2^30 bytes), plus a 5 GB margin. # Use the destination's filesystem, creating the directory when it is new. -# The bytes of ARCHIVE, or of its interrupted ARCHIVE.part, are already part -# of the peak, so a resumed download needs only the rest. +# Allocated blocks of ARCHIVE, or of its interrupted ARCHIVE.part, are already +# part of the peak, so a resumed download needs only the rest. # An unreadable df is a failed preflight, never permission to fill the disk. clinical_disk_ok() { local dir=$1 peak=$2 label=$3 archive=${4:-} free have=0 f mkdir -p "$dir" || return 1 + # A stopped extraction is disposable. Reclaim it before measuring space, + # otherwise its staging tree can prevent every retry from reaching cleanup. + rm -rf "${dir}"/.extract.* || return 1 free=$(df -Pk "$dir" 2>/dev/null | awk 'NR == 2 {print $4}') || free="" case "$free" in ''|*[!0-9]*) echo "[WARN] Cannot read free space for ${label}; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; return 1 ;; @@ -676,7 +679,10 @@ clinical_disk_ok() { if [ -n "$archive" ]; then for f in "$archive" "${archive}.part"; do if [ -f "$f" ]; then - have=$(( $(wc -c < "$f") / 1024 )) + have=$(du -k "$f" 2>/dev/null | awk 'NR == 1 {print $1}') || have="" + case "$have" in + ''|*[!0-9]*) echo "[WARN] Cannot read allocated space for ${label}; retry the clinical install."; return 1 ;; + esac break fi done @@ -704,7 +710,12 @@ install_pcgr_bundle() { url="https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" tarball="${dir}/${url##*/}" mkdir -p "$dir" || return 1 - # fetch publishes the archive only after its check, so one already here is whole. + # A retained archive may have changed since fetch checked it. Verify it + # against the pin again before reuse, replacing a mismatch through fetch. + if [ -f "$tarball" ] && ! _fetch_check sha256 "$PCGR_DATA_SHA256" "$tarball" "$tarball" "${url##*/}"; then + echo "[WARN] Removing PCGR/CPSR archive that failed its checksum; downloading it again." + rm -f "$tarball" || return 1 + fi [ -f "$tarball" ] || fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 rm -rf "${dir}"/.extract.* tmp=$(mktemp -d "${dir}/.extract.XXXXXX") || return 1 From b7e3f47c4f3155cc04e7c51b384f7c8f9ed4d551 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:24:04 +0100 Subject: [PATCH 80/86] fix(live): keep the reader's igv place through a resize, and document the reads level A resize applied the URL's locus again even after the reader moved inside igv, so any resize threw the reader back. The viewer now follows the URL only until the reader pans, zooms or types a locus, and again after the next locus from the map. docs/genome-map.md now names the Reads stop, the R key, the Esc order and the card's See the reads button. --- CHANGELOG.md | 2 +- assets/viewer/viewer.js | 22 +++++++++++++++++++--- docs/genome-map.md | 15 ++++++++++----- docs/viewer.md | 2 +- tests/test_live_viewer.py | 18 ++++++++++++++++++ 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de2d8186..1ea353a9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus until you move inside the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index 558ea23cd..7edf7248a 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -64,12 +64,28 @@ browser.search(locus); } } - window.addEventListener("hashchange", followLocus); - // igv.js keeps bases per pixel on resize. Keep the requested absolute locus instead. + // igv.js keeps bases per pixel on resize. While igv still shows the locus the URL asks + // for, a resize applies that locus again. Once the reader pans, zooms or types a locus in + // igv, igv keeps the reader's place, until the URL asks for a new locus. The events + // come out of igv's shadow root, because pointer, wheel and key events are composed. + let follow = true; + ["pointerdown", "wheel", "keydown"].forEach(function (type) { + document.getElementById("igv").addEventListener(type, function () { + follow = false; + }, { capture: true, passive: true }); + }); + window.addEventListener("hashchange", function () { + follow = true; + followLocus(); + }); let resizeTimer; window.addEventListener("resize", function () { window.clearTimeout(resizeTimer); - resizeTimer = window.setTimeout(followLocus, 100); + resizeTimer = window.setTimeout(function () { + if (follow) { + followLocus(); + } + }, 100); }); root.dataset.pgpTracks = String(cfg.tracks.length); root.dataset.pgpStatus = "ready"; diff --git a/docs/genome-map.md b/docs/genome-map.md index f7e3296c1..413c93064 100644 --- a/docs/genome-map.md +++ b/docs/genome-map.md @@ -12,7 +12,7 @@ The map has five levels. The level follows how many bases one screen pixel cover | Chromosome | more than 16,667 bases per pixel | one chromosome open in place: an overview with the part in view, a ruler, the bands with their names, a findings lane with each gene and its level's words, and the runs of homozygosity with their lengths | | Region | 833 to 16,667 bases per pixel | the same lanes, with each band named with its chromosome (`16p13.3`), and the genes lane: every gene that fits in three rows, its name beside it; the genes a finding names come first, in bold, then the protein-coding genes, then the rest | | Gene | 833 bases per pixel or fewer, a view of 50 kb or more | the genes as models: the span as a thin line with its strand's chevrons and the exon blocks inside it; each finding is a dashed tick down through every lane, with its change (HGVS) under its gene | -| Reads | a view under 50 kb | drawn as the gene level | +| Reads | a view under 50 kb | drawn as the gene level, with a brass band under the ruler, a line that says the reads are one key away, and a **See the reads** button on the map | A dashed line marks the centre of an open chromosome. The margin beside the map says what is under it: the chromosome, the position and the band, the findings in view and the runs of homozygosity. @@ -21,9 +21,9 @@ The exon blocks are the union of the exons of every basic GENCODE transcript of ## Moving around - Click a chromosome to open it, and drag to move along it. Scroll, the + and − buttons or the + and − keys zoom. -- The zoom stops above the map (Genome, Chromosome, Region, Gene) jump to a level and keep the place in view. From the whole genome they go to the first finding by level. +- The zoom stops above the map (Genome, Chromosome, Region, Gene, Reads) jump to a level and keep the place in view. From the whole genome they go to the first finding by level. Reads opens a 2 kb view and the reads panel. - The path under the title (`Genome › chr16 › 16p13.3 › 2.050-2.300M`) shows where the map is; each step up is a button. -- Click a gene to zoom to it. Click a finding to open its card in the margin: its level, title, gene, position, change and the map's sentence about it, with a link to the finding's own page. +- Click a gene to zoom to it. Click a finding to open its card in the margin: its level, title, gene, position, change and the map's sentence about it, with a link to the finding's own page and a **See the reads** button. - When an open chromosome has no finding in view, a "Nearest" button names the closest finding on that chromosome, or the first one on the next chromosome that has one, and flies there. - The address follows the map: `#genome`, `#chr7`, `#chr7:117.4-137.4M`, `#chr7:117.480-117.669M` or `#chr7:117559001-117599000` (1-based, both ends included). A link to one of them opens that view, and Back returns to the view before. @@ -36,12 +36,17 @@ The exon blocks are the union of the exons of every basic GENCODE transcript of | + − | zoom in or out 2×, at the centre | | `[` `]` | the chromosome before or after | | `0` | the whole genome | -| Esc | closes the card, then steps out one level: reads to gene, gene to region, region to the whole chromosome, chromosome to the genome | +| Esc | closes the reads panel, then the card, then steps out one level: reads to gene, gene to region, region to the whole chromosome, chromosome to the genome | | Tab | steps through the findings in view, each with a ring on the map and its card, then leaves the map | | `L` | opens the layers and moves to them | | `T` | folds or opens the In view table | +| `R` | opens the reads panel, in a view under 50 kb | -The keys work with the map focused (`/`, `L` and `T` anywhere on the page) and never inside a text box or with Cmd, Ctrl or Alt. +The keys work with the map focused (`/`, `L`, `T` and `R` anywhere on the page) and never inside a text box or with Cmd, Ctrl or Alt. + +### The reads + +Under 50 kb, the run's reads open in a panel under the map, in the genome viewer. `docs/viewer.md` describes the panel. ### The whole-genome strip diff --git a/docs/viewer.md b/docs/viewer.md index 09b8e3680..add80039b 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -84,7 +84,7 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. -Back follows the map's history. Moving the map creates no extra history entry for the iframe, and the Reads gauge adds one camera view. Resizing the viewer keeps its requested locus. +Back follows the map's history. Moving the map creates no extra history entry for the iframe, and the Reads gauge adds one camera view. Resizing the viewer keeps the requested locus. Once you pan, zoom or type a locus inside the viewer, a resize keeps your place instead, until the map moves again. The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. Closing the panel clears its previous locus, title, and link to the full viewer. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 666a8b9f0..2906d191f 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -313,6 +313,24 @@ def resized(): out.write(base64.b64decode(png)) b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, b.session) + # A locus the reader types in igv stays through a resize: only a new map locus moves igv again. + js("var i=savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input');" + "i.focus();i.select();") + b.call("Input.insertText", {"text": "chr1:2001-3000"}, b.session) + for kind in ("keyDown", "keyUp"): + b.call("Input.dispatchKeyEvent", {"type": kind, "key": "Enter", "code": "Enter", + "windowsVirtualKeyCode": 13, "text": "\r"}, b.session) + typed = b.wait("/^chr1:2,0/.test(savedFrame.contentDocument.getElementById('igv').shadowRoot" + ".querySelector('input.igv-search-input').value)") + js("savedFrame.style.width='50%'") + resized() + time.sleep(0.5) # past viewer.js's 100 ms resize timer + m = re.match(r"chr1:([\d,]+)-([\d,]+)$", viewer_locus() or "") + check("viewer: a locus typed in igv stays through a resize", + typed and m and 1500 <= int(m.group(1).replace(",", "")) <= 3000 + and int(m.group(2).replace(",", "")) <= 3500, viewer_locus()) + js("savedFrame.style.width=''") + resized() b.eval("savedFrame.contentDocument.dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") check("map: Escape closes the reads and returns focus without moving the map", b.eval("document.querySelector('.gm-reads').hidden && document.activeElement===document.querySelector('.gm-canvas') && " From 3b223311d26045e507112d04c10b30a26a5f3221 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:25:32 +0100 Subject: [PATCH 81/86] test(bloodtype): pin split null allele dosage and contradictions Bead: trait-40 Red first: all three new report tests fail on the pre-fix accumulator. --- rust/crates/og-bloodtype/tests/report.rs | 96 ++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index 6cb81053f..165326f5a 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -427,3 +427,99 @@ fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { assert_eq!(status, "absent", "{recs:?}"); } } + +/// The complete table and evidence must agree through both VCF readers. +fn report_both_ways(d: &Path, records: &[String]) -> (String, String) { + let (plain, gz) = both_ways(d, records); + let mut outputs = Vec::new(); + for (vcf, access) in [(plain, Access::Stream), (gz, Access::Index)] { + let files = Files { + vcf: vcf.to_string_lossy().into_owned(), + markers: markers(), + depth: Some(depth(d, 2.0, &HashMap::new())), + cn_loci: Some(cn_loci()), + out: d.join("table.tsv").to_string_lossy().into_owned(), + evidence: d.join("evidence.tsv").to_string_lossy().into_owned(), + }; + assert_eq!(report(&files).unwrap().1, access); + outputs.push(( + std::fs::read_to_string(&files.out).unwrap(), + std::fs::read_to_string(&files.evidence).unwrap(), + )); + } + assert_eq!(outputs[0], outputs[1]); + outputs.remove(0) +} + +fn split_null_marker(name: &str, system: &str, phenotype: &str) { + let d = dir(&format!("split-{name}")); + let m = read_markers(&markers()).unwrap()[name].clone(); + let alts: Vec<&str> = m.alt.split(',').collect(); + let record = |alt: &str, gt: &str| line(&m.chrom, m.pos, &m.ref_, alt, "PASS", gt); + let unsplit = record(&m.alt, "1/2"); + let expected = report_both_ways(&d, std::slice::from_ref(&unsplit)); + let row: Vec<&str> = expected.0.lines().find(|l| l.starts_with(&format!("{system}\t"))) + .unwrap().split('\t').collect(); + assert_eq!(row[..4], [system, phenotype, "null/null", "high"]); + let a = record(alts[0], "1/0"); + let b = record(alts[1], "0/1"); + for records in [ + vec![a.clone(), b.clone()], + vec![b.clone(), a.clone()], + vec![a.clone(), a.clone(), b.clone(), b.clone()], + vec![unsplit.clone(), a.clone(), b.clone()], + vec![b.clone(), a.clone(), unsplit], + vec![record(&format!("{},{}", alts[1], alts[0]), "2/1")], + vec![record(".", "0/0"), a, b], + ] { + assert_eq!(report_both_ways(&d, &records), expected, "{records:?}"); + } + // A repeated call of one ALT is still one copy, even when another + // record lists the same ALT at a different allele index. + let one = record(alts[0], "0/1"); + for gt in ["0/1", "1/1"] { + let single = record(alts[0], gt); + let expected = report_both_ways(&d, std::slice::from_ref(&single)); + let copies = if gt == "0/1" { 1 } else { 2 }; + let reordered = record(&format!("{},{}", alts[1], alts[0]), + if copies == 1 { "0/2" } else { "2/2" }); + let records = [single.clone(), reordered, single]; + assert_eq!(report_both_ways(&d, &records), expected); + assert_eq!(site_both_ways(&d, &records, name), ("called".to_owned(), Some(copies), String::new())); + } + assert_eq!(site_both_ways(&d, &[one], name).1, Some(1)); +} + +#[test] +fn split_kidd_null_alts_match_an_unsplit_call() { + split_null_marker("JK_01N06", "Kidd", "Jk(a-b-)"); +} + +#[test] +fn split_kell_null_alts_match_an_unsplit_call() { + split_null_marker("KEL_02N01", "Kell", "K-k- Kp(a-b-) Js(a-b-)"); +} + +#[test] +fn incompatible_null_alt_dosages_are_no_call_in_both_readers() { + for name in ["JK_01N06", "KEL_02N01"] { + let d = dir(&format!("incompatible-{name}")); + let m = read_markers(&markers()).unwrap()[name].clone(); + let alts: Vec<&str> = m.alt.split(',').collect(); + let record = |alt: &str, gt: &str| line(&m.chrom, m.pos, &m.ref_, alt, "PASS", gt); + // Two distinct ALTs cannot have three copies. Different dosages + // for the same ALT cannot both be true either. + let hom = record(alts[0], "1/1"); + for other in [record(alts[1], "0/1"), record(alts[0], "0/1")] { + for records in [vec![hom.clone(), other.clone()], vec![other.clone(), hom.clone()]] { + let (status, n, note) = site_both_ways(&d, &records, name); + assert_eq!((status.as_str(), n), ("no call", None), "{records:?}"); + assert_eq!(note, "incompatible ALT dosages at this position"); + let (table, _) = report_both_ways(&d, &records); + let row: Vec<&str> = table.lines().find(|l| l.starts_with(&format!("{}\t", m.system))) + .unwrap().split('\t').collect(); + assert_ne!(row[3], "high"); + } + } + } +} From 0a0cfc073de74574954914a2872ce96e280152e9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:31:47 +0100 Subject: [PATCH 82/86] fix(roh): write 0.0, not -0.0, when a sample has no long run An empty f64 sum in Rust is -0.0, so every sample without a run of 5 Mb or more read F_ROH -0.0000 in both reports. Fold from +0.0, check the sign in the scope test, and keep the ROH file's open and read errors named the way every other section names them. --- rust/crates/og-summary/src/collect/group_c/roh.rs | 13 ++++++++++--- rust/crates/og-summary/src/collect/group_c/tests.rs | 9 +++++++++ .../golden/summary-collect/all-bad/all-bad.json | 2 +- .../parity/golden/summary-collect/c-bad1/bad1.json | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index 11a538c28..e0790620c 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -223,11 +223,16 @@ pub fn section(c: &Ctx<'_>) -> Out { // chrom -> the bases of its long runs let mut per_chrom: indexmap::IndexMap = indexmap::IndexMap::new(); let (mut assessed, mut observed) = ([false; 22], [false; 22]); - let mut reader = crate::text_lines(&p.path)?; + let mut reader = crate::text_lines(&p.path).map_err(|e| super::util::os_note(&e))?; let mut line = String::new(); loop { line.clear(); - if !reader.read_line(&mut line).map_err(|e| e.to_string())? { + if !reader.read_line(&mut line).map_err(|e| { + crate::collect::sections::py_error( + "OSError", + &crate::collect::sections::gzip_message(&e), + ) + })? { break; } let c: Vec<&str> = line.split_whitespace().collect(); @@ -285,7 +290,9 @@ pub fn section(c: &Ctx<'_>) -> Out { big.push(row.val()); } } - let long_bp: f64 = per_chrom.values().sum(); + // fold from +0.0: an empty f64 sum is -0.0, which the reports print as + // "-0.0000" for every sample without a long run. + let long_bp: f64 = per_chrom.values().fold(0.0, |a, v| a + v); let has_sites = assessed.iter().any(|yes| *yes); // Old -O r files omit ST rows. Their runs give a lower bound on // chromosome scope, never evidence for a genome-wide interpretation. diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index e4cad1c0e..c50a4f5dc 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -268,6 +268,15 @@ fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { let no_runs = roh_scope_summary(&[20], ""); assert_eq!(at(&no_runs, &["f_roh"]), &Value::Float(0.0)); assert_eq!(at(&no_runs, &["relationship"]), &Value::Null); + // Value::Float(-0.0) == Value::Float(0.0), so check the sign apart: a + // sample with no long run must read 0.0, never -0.0, in both reports. + for key in ["f_roh", "long_mb"] { + assert!( + matches!(at(&no_runs, &[key]), Value::Float(f) if f.is_sign_positive()), + "{key} must be +0.0 without long runs, got {:?}", + at(&no_runs, &[key]) + ); + } } #[test] diff --git a/tests/parity/golden/summary-collect/all-bad/all-bad.json b/tests/parity/golden/summary-collect/all-bad/all-bad.json index a20f67e6c..24bc28580 100644 --- a/tests/parity/golden/summary-collect/all-bad/all-bad.json +++ b/tests/parity/golden/summary-collect/all-bad/all-bad.json @@ -525,7 +525,7 @@ "autosomal_over_5mb": [], "masked_mb": 0.0, "sex_chromosome_segments": 0, - "long_mb": -0.0, + "long_mb": 0.0, "f_roh": null, "relationship": null, "upd_flag": null, diff --git a/tests/parity/golden/summary-collect/c-bad1/bad1.json b/tests/parity/golden/summary-collect/c-bad1/bad1.json index ad14386d1..8bf1c4c9e 100644 --- a/tests/parity/golden/summary-collect/c-bad1/bad1.json +++ b/tests/parity/golden/summary-collect/c-bad1/bad1.json @@ -514,7 +514,7 @@ "autosomal_over_5mb": [], "masked_mb": 0.0, "sex_chromosome_segments": 0, - "long_mb": -0.0, + "long_mb": 0.0, "f_roh": null, "relationship": null, "upd_flag": null, From 26f7be164db42fa58d37d879f97baa1015910331 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:32:58 +0100 Subject: [PATCH 83/86] fix(bloodtype): combine distinct null ALTs across split records Keep called ALT identities, deduplicate repeated dosages and refuse conflicting or impossible totals. Both VCF readers now preserve split/unsplit tables and evidence. Bead: trait-40 --- CHANGELOG.md | 1 + docs/41-bloodtype.md | 1 + rust/crates/og-bloodtype/src/genotypes.rs | 42 ++++++++++++++++++++++- rust/crates/og-bloodtype/tests/report.rs | 33 +++++++++++++----- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab677d5bb..f30350c3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Step 41 (blood groups) reads every VCF record that covers a marker site, and their order no longer changes the answer. Before, the first record at a position won. An all-sites VCF writes a reference record and then the indel at one position (`T . 0/0`, then `T TC 0/1` at ABO c.261), so a person with an A or B allele read O. A split multi-allelic record with the SNV first did the same. No column, file or option changes; only the values of the affected sites do. What changes: - A call of the marker's ALT now wins over the other records at the position. Another allele a second record calls there is named in the note, as when one record holds both. + - Distinct ALTs listed by one null marker add their copies across split records. JK_01N06 and KEL_02N01 now give the same evidence and null/null phenotype before and after splitting. Repeated calls of the same ALT count once. Conflicting dosages for one ALT, or totals above two copies, give a no call with an incompatible-dosage note. - A no call of a record that names the marker's ALT (`./.`, or a failed FILTER) now wins over a reference record at the same position, so the site is a no call instead of a confident reference. - A 0/0 record over a REF shorter than the marker's (an all-sites `C . 0/0`, or a gVCF block that starts at the position) now counts as no record, so the reads decide. Before it was a no call: ABO read "A, A1 or A2 not determined", and Vel, HPA-14 and every system with a deletion marker lost confidence. - A record that starts upstream and spans a marker with an allele that changes the marker's base (a deletion over it, or a padded `CG CA`, as DeepVariant writes `CG C,CA` when a deletion and an SNV overlap) now makes the site a no call, with a note that names the record. Before, the site had no record at its own position and 8 reads made it a confident reference: a K/K person read k/k at high confidence. diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 2456d51b5..9f1941391 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -67,6 +67,7 @@ A site the VCF has no record for is the reference allele when the BAM has at lea When several records cover one site, each one is read and their order does not matter: - A call of the table's ALT wins over the other records at the site's position. An all-sites VCF writes `T . 0/0` and then `T TC 0/1` at ABO c.261, so the insertion counts. Another allele that a second record calls there (a split `T G 1/0`) is named in the note. +- When one null marker lists several ALTs, calls of distinct listed ALTs add their copies. Splitting the record keeps the same evidence and phenotype. Repeated calls of the same ALT count once, even if its allele index changes. Different dosages for the same ALT, or a total above two copies, make the site a no call with an incompatible-dosage note. - A no call of a record that names the table's ALT (`./.`, or a failed FILTER) comes next: the ALT may be there, so a reference record beside it does not make the site reference. - Then a reference call, then any other no call. - A record that starts upstream and spans the site with an allele that changes the site's base (a deletion over it, or a padded record such as `CG CA`) makes the site a no call, whatever the site's own records say. A reference call or a no call upstream changes nothing, and neither does an allele that keeps the site's base. diff --git a/rust/crates/og-bloodtype/src/genotypes.rs b/rust/crates/og-bloodtype/src/genotypes.rs index 6eac1aec9..f2506464b 100644 --- a/rust/crates/og-bloodtype/src/genotypes.rs +++ b/rust/crates/og-bloodtype/src/genotypes.rs @@ -22,6 +22,8 @@ //! ALT wins, then a no call of a record that names the marker's ALT, //! then a reference call, then any other no call; the first of equals //! counts, and another allele a losing call names stays in the note; +//! - called copies of distinct ALTs listed by one marker add up; repeated +//! calls of the same ALT count once, and incompatible dosages are no call; //! - a reference-only record whose REF is shorter than the marker's (an //! all-sites record or a gVCF block starting at a deletion marker) says //! nothing past its own bases, so the site reads as if it had no record. @@ -174,6 +176,8 @@ struct Reading { status: Status, n: Option, note: String, + /// Called marker ALTs and their dosages, independent of VCF allele indices. + alts: BTreeMap, } impl Reading { @@ -183,6 +187,7 @@ impl Reading { status: Status::NoCall, n: None, note, + alts: BTreeMap::new(), } } } @@ -196,10 +201,23 @@ struct Evidence { others: Vec, /// The first upstream record that changes the site's base. span: Option, + /// One dosage per called marker ALT, bounded by the marker's ALT list. + alts: BTreeMap, + /// Different calls of one ALT disagree about its dosage. + incompatible: bool, } impl Evidence { fn add(&mut self, r: Reading) { + for (alt, &copies) in &r.alts { + match self.alts.get(alt) { + Some(&previous) if previous != copies => self.incompatible = true, + Some(_) => {} + None => { + self.alts.insert(alt.clone(), copies); + } + } + } let loser = match &mut self.best { Some(b) if r.weight > b.weight => Some(std::mem::replace(b, r)), Some(_) => Some(r), @@ -224,8 +242,19 @@ impl Evidence { site.n = None; site.note = note; } else if let Some(b) = self.best { + let copies: i64 = self.alts.values().sum(); + if self.incompatible || copies > 2 { + site.status = Status::NoCall; + site.n = None; + site.note = "incompatible ALT dosages at this position".to_owned(); + return; + } site.status = b.status; - site.n = b.n; + site.n = if b.weight == Weight::Alt { + Some(copies) + } else { + b.n + }; site.note = if b.status == Status::Called { let others = self.others.into_iter().filter(|o| *o != b.note); std::iter::once(b.note.clone()) @@ -383,6 +412,16 @@ fn read_record( .iter() .filter(|a| idx.iter().any(|i| i == *a)) .count() as i64; + let mut called_alts = BTreeMap::new(); + for (i, alt) in alts.iter().enumerate() { + if wants.contains(alt) { + let index = (i + 1).to_string(); + let copies = alleles.iter().filter(|a| **a == index).count() as i64; + if copies > 0 { + *called_alts.entry((*alt).to_owned()).or_default() += copies; + } + } + } let note = if alleles .iter() .any(|a| *a != "0" && !idx.iter().any(|i| i == a)) @@ -403,6 +442,7 @@ fn read_record( status: Status::Called, n: Some(n), note, + alts: called_alts, }) } diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index 165326f5a..43de618cf 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -458,8 +458,13 @@ fn split_null_marker(name: &str, system: &str, phenotype: &str) { let record = |alt: &str, gt: &str| line(&m.chrom, m.pos, &m.ref_, alt, "PASS", gt); let unsplit = record(&m.alt, "1/2"); let expected = report_both_ways(&d, std::slice::from_ref(&unsplit)); - let row: Vec<&str> = expected.0.lines().find(|l| l.starts_with(&format!("{system}\t"))) - .unwrap().split('\t').collect(); + let row: Vec<&str> = expected + .0 + .lines() + .find(|l| l.starts_with(&format!("{system}\t"))) + .unwrap() + .split('\t') + .collect(); assert_eq!(row[..4], [system, phenotype, "null/null", "high"]); let a = record(alts[0], "1/0"); let b = record(alts[1], "0/1"); @@ -481,11 +486,16 @@ fn split_null_marker(name: &str, system: &str, phenotype: &str) { let single = record(alts[0], gt); let expected = report_both_ways(&d, std::slice::from_ref(&single)); let copies = if gt == "0/1" { 1 } else { 2 }; - let reordered = record(&format!("{},{}", alts[1], alts[0]), - if copies == 1 { "0/2" } else { "2/2" }); + let reordered = record( + &format!("{},{}", alts[1], alts[0]), + if copies == 1 { "0/2" } else { "2/2" }, + ); let records = [single.clone(), reordered, single]; assert_eq!(report_both_ways(&d, &records), expected); - assert_eq!(site_both_ways(&d, &records, name), ("called".to_owned(), Some(copies), String::new())); + assert_eq!( + site_both_ways(&d, &records, name), + ("called".to_owned(), Some(copies), String::new()) + ); } assert_eq!(site_both_ways(&d, &[one], name).1, Some(1)); } @@ -511,13 +521,20 @@ fn incompatible_null_alt_dosages_are_no_call_in_both_readers() { // for the same ALT cannot both be true either. let hom = record(alts[0], "1/1"); for other in [record(alts[1], "0/1"), record(alts[0], "0/1")] { - for records in [vec![hom.clone(), other.clone()], vec![other.clone(), hom.clone()]] { + for records in [ + vec![hom.clone(), other.clone()], + vec![other.clone(), hom.clone()], + ] { let (status, n, note) = site_both_ways(&d, &records, name); assert_eq!((status.as_str(), n), ("no call", None), "{records:?}"); assert_eq!(note, "incompatible ALT dosages at this position"); let (table, _) = report_both_ways(&d, &records); - let row: Vec<&str> = table.lines().find(|l| l.starts_with(&format!("{}\t", m.system))) - .unwrap().split('\t').collect(); + let row: Vec<&str> = table + .lines() + .find(|l| l.starts_with(&format!("{}\t", m.system))) + .unwrap() + .split('\t') + .collect(); assert_ne!(row[3], "high"); } } From 6af4386a2dc7e922fc7a94a3d6dbcf0ce5795467 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:37:21 +0100 Subject: [PATCH 84/86] fix(recheck): replay only the switches run-all.sh records, and refuse a ~ bash would expand An edited logs/run_status.tsv could name BASH_ENV, PATH or LD_PRELOAD in both 'meta switches' and 'meta command', and the replay then set it for the bash it starts. A switch is now a name in run-all.sh's SWITCHES list too (a unit test keeps the Rust copy equal to it). A ~ after an unquoted = or : is refused, since bash expands it there and printf %q never writes it so. --- docs/recheck.md | 4 +- rust/crates/og-recheck/src/rerun.rs | 79 ++++++++++++++++++++++++++--- rust/crates/og-recheck/src/tests.rs | 33 ++++++++++++ 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/docs/recheck.md b/docs/recheck.md index bfc471eb2..3f5f93c52 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -68,7 +68,7 @@ bin/og recheck --sample your_sample --rerun bin/og recheck --sample your_sample --rerun --refresh-clinvar # refresh ClinVar first ``` -`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` reads that command back word by word: the `cd` folder, the switch assignments, the `run-all.sh` path and its arguments. It starts that `run-all.sh` with `bash`, in that folder, with the arguments one by one and those switches only, so a `TOOLS` set in your shell today does not change the run. No shell reads the command. A row of any other shape, for example one edited to add `; rm ...`, is refused with exit 2 and nothing runs; a `cd` folder that no longer exists is refused the same way. Check such a row, then run the command by hand. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. +`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` reads that command back word by word: the `cd` folder, the switch assignments, the `run-all.sh` path and its arguments. It starts that `run-all.sh` with `bash`, in that folder, with the arguments one by one and those switches only, so a `TOOLS` set in your shell today does not change the run. A switch is a name in the `SWITCHES` list of `run-all.sh`; an assignment to any other name (`BASH_ENV`, `PATH`, `LD_PRELOAD`) is refused, even when the `meta switches` line names it. No shell reads the command. A row of any other shape, for example one edited to add `; rm ...`, is refused with exit 2 and nothing runs; a `cd` folder that no longer exists is refused the same way. Check such a row, then run the command by hand. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. It refuses while a run of the sample is going on (`run_status.tsv` has no `finished_utc`), and while another `--rerun` of the sample holds `recheck/rerun.lock`. A run that was killed without its exit trap leaves no `finished_utc` either: run `run-all.sh` by hand once, and the next check can rerun again. @@ -170,4 +170,4 @@ launchd runs an interval missed during sleep when the Mac wakes; one missed whil ## Tests -`cargo test -p og-recheck` checks the classes, the steps each item names, a failed lookup (an offline answer and a server on 127.0.0.1), the files it writes, the refusals during a run and while a lock is held, the lock held while `setup.sh` refreshes ClinVar and let go before the replay, the comparison after a rerun and `--launch-dir`. `tests/test_owngenome_recheck.py` runs `og recheck` from the outside, the deprecated `bin/owngenome_recheck.py`, the live-site plugin and the ClinVar lock block of `run-all.sh` and step 06, each against a planted mistake too. `tests/fake-docker/run-all-recheck.sh` checks that `--rerun` replays the recorded command exactly, with a control where one dropped argument makes it differ. +`cargo test -p og-recheck` checks the classes, the steps each item names, a failed lookup (an offline answer and a server on 127.0.0.1), the files it writes, the refusals during a run and while a lock is held, the lock held while `setup.sh` refreshes ClinVar and let go before the replay, the comparison after a rerun, `--launch-dir`, the last line when lookups fail (connections refused), a recorded command read back word by word and refused when it is not a `run-all.sh` call, and the replay with typed arguments. `tests/test_owngenome_recheck.py` runs `og recheck` from the outside, the deprecated `bin/owngenome_recheck.py`, the live-site plugin and the ClinVar lock block of `run-all.sh` and step 06, each against a planted mistake too. `tests/fake-docker/run-all-recheck.sh` checks that `--rerun` replays the recorded command exactly, with a control where one dropped argument makes it differ. diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index af72f1474..9a1bd4f34 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -121,6 +121,58 @@ pub enum Rerun { Refused(u8, String), } +/// The switches `scripts/run-all.sh` records (its `SWITCHES` array; a unit +/// test keeps the two equal). A replay sets or drops only these, whatever +/// `meta switches` says, so an edited row can never hand `BASH_ENV`, `PATH`, +/// `LD_PRELOAD` or `DYLD_*` to the `bash` it starts. +pub const SWITCHES: &[&str] = &[ + "GENOME_DIR", + "SKIP_VALIDATION", + "THREADS", + "SKIP_TRIM", + "INTERVALS", + "ALIGN_DIR", + "TOOLS", + "KIR", + "HLA_QC", + "HPO", + "REF_FASTA", + "EH_CATALOG", + "MANTA_CALL_REGIONS", + "PARASCOPY_POPULATION", + "PARAPHASE_REGIONS", + "ANCESTRY_PANEL", + "ANCESTRY_FINE_DIR", + "ARCHAIC_DIR", + "ARCHAIC_POP", + "PGSC_CALC_DIR", + "ANCIENT_PANEL_DIR", + "VIRAL_REF", + "MEGANE_KMERS", + "MOCHA_DATA", + "GRIDSS", + "IMPUTATION", + "SOMATIC", + "EXTRA_CALLERS", + "BENCHMARK", + "FAMILIES", + "OWNGENOME_LIVE_URL", + "LOCITYPER_DATA", + "LOCITYPER_BG_REGION", + "NXF_VER", + "NXF_WORK", + "NXF_HOME", + "NXF_OFFLINE", +]; + +/// The names of `meta switches` that are also in [`SWITCHES`]. +fn known(switches: &str) -> Vec<&str> { + switches + .split_whitespace() + .filter(|n| SWITCHES.contains(n)) + .collect() +} + /// One shell word of a recorded command. #[derive(Debug, Clone, PartialEq, Eq)] enum Token { @@ -131,7 +183,8 @@ enum Token { } /// Bytes `printf %q` leaves unquoted anywhere in a word, and that mean -/// nothing to the shell there. `#` and `~` do too, except first in a word. +/// nothing to the shell there. `#` and `~` do too, except first in a word +/// (and `~` after an unquoted `=` or `:`, see [`words`]). fn plain(b: u8, first: bool) -> bool { b.is_ascii_alphanumeric() || b"%+-./:=@_,".contains(&b) @@ -200,7 +253,8 @@ fn ansi_c(s: &[u8], i: &mut usize, out: &mut Vec) -> Option<()> { /// plain bytes, `\x`, `'...'` and `$'...'`, words split on spaces and tabs, /// and the `&&` operator. `None` for anything else a shell would act on (a /// `;`, a pipe, a redirect, an expansion, a glob, a double quote, a line -/// break) and for text that is not UTF-8. +/// break, a `~` first in a word or after an unquoted `=` or `:`, where bash +/// expands it) and for text that is not UTF-8. fn words(cmd: &str) -> Option> { let s = cmd.as_bytes(); let (mut out, mut i) = (Vec::new(), 0); @@ -215,6 +269,8 @@ fn words(cmd: &str) -> Option> { continue; } let (mut w, mut lead, mut quoted) = (Vec::new(), 0, false); + // The byte before, when it stood unquoted. + let mut prev: Option = None; while i < s.len() && !matches!(s[i], b' ' | b'\t') { let b = s[i]; if b == b'\\' { @@ -226,17 +282,23 @@ fn words(cmd: &str) -> Option> { w.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); i += 1 + ch.len_utf8(); quoted = true; + prev = None; } else if b == b'\'' { let end = s[i + 1..].iter().position(|&c| c == b'\'')? + i + 1; w.extend_from_slice(&s[i + 1..end]); i = end + 1; quoted = true; + prev = None; } else if b == b'$' && s.get(i + 1) == Some(&b'\'') { i += 2; ansi_c(s, &mut i, &mut w)?; quoted = true; - } else if plain(b, w.is_empty() && !quoted) { + prev = None; + } else if plain(b, w.is_empty() && !quoted) + && !(b == b'~' && matches!(prev, Some(b'=' | b':'))) + { w.push(b); + prev = Some(b); i += 1; if !quoted { lead = w.len(); @@ -265,10 +327,10 @@ pub struct Call { /// `meta command` as a typed call, or `None` when it is not the shape /// `run-all.sh` writes: `cd DIR && [NAME=VALUE ...] .../scripts/run-all.sh -/// ARG ...`, every `NAME` one of `switches`. Nothing of it reaches a -/// shell. +/// ARG ...`, every `NAME` one of `switches` and of [`SWITCHES`]. Nothing of +/// it reaches a shell. pub fn parse_call(cmd: &str, switches: &str) -> Option { - let names: Vec<&str> = switches.split_whitespace().collect(); + let names = known(switches); let mut t = words(cmd)?.into_iter(); let mut word = || match t.next() { Some(Token::Word(w, lead)) => Some((w, lead)), @@ -314,7 +376,8 @@ pub fn parse_call(cmd: &str, switches: &str) -> Option { /// Replays the last `run-all.sh` command of the sample (`meta command` of /// `logs/run_status.tsv`) with the same switches: the switch variables the -/// run read (`meta switches`) are dropped from the environment, so the +/// run read (`meta switches`, only names in [`SWITCHES`]) are dropped from +/// the environment, so the /// command's own assignments decide. The command is read as a typed call /// ([`parse_call`]) and `run-all.sh` is started by `bash` with its /// arguments one by one; a command of any other shape is refused and @@ -381,7 +444,7 @@ pub fn rerun( append_log(log, &[[&started, "rerun-start", "", "", &cmd]])?; let mut c = Command::new("bash"); c.arg(&call.script).args(&call.args).current_dir(&call.dir); - for name in switches.split_whitespace() { + for name in known(&switches) { c.env_remove(name); } for (name, value) in &call.env { diff --git a/rust/crates/og-recheck/src/tests.rs b/rust/crates/og-recheck/src/tests.rs index a19d8dd4f..c0c4ac869 100644 --- a/rust/crates/og-recheck/src/tests.rs +++ b/rust/crates/og-recheck/src/tests.rs @@ -1262,9 +1262,42 @@ fn a_recorded_command_is_read_as_a_typed_call() { "cd /d /o/scripts/run-all.sh S1 male".to_owned(), "\\cd /d && /o/scripts/run-all.sh S1 male".to_owned(), "cd /d && 'TOOLS=x' /o/scripts/run-all.sh S1".to_owned(), + // bash expands a ~ after an unquoted = or :, printf %q writes \~. + format!("{base} a=~b"), + format!("{base} a:~b"), + format!("{base} --x=a:~/y"), + "cd /d && TOOLS=~/x /o/scripts/run-all.sh S1 male".to_owned(), ] { assert_eq!(parse_call(&bad, "TOOLS"), None, "{bad}"); } + // printf %q's own escape of such a ~ reads back as the ~ itself. + let call = parse_call(&format!("{base} a=\\~b a:\\~b"), "TOOLS").unwrap(); + assert_eq!(call.args[2..], ["a=~b", "a:~b"]); + // A name only an edited 'meta switches' line names is not a switch: + // the replay never hands it to bash. + for name in [ + "BASH_ENV", + "PATH", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + "ENV", + ] { + let cmd = format!("cd /d && {name}=/tmp/x TOOLS=a /o/scripts/run-all.sh S1 male"); + assert_eq!(parse_call(&cmd, &format!("TOOLS {name}")), None, "{cmd}"); + } +} + +#[test] +fn the_switches_a_replay_may_set_are_the_ones_run_all_records() { + let text = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../scripts/run-all.sh"), + ) + .unwrap(); + let start = text.find("\nSWITCHES=(").expect("SWITCHES in run-all.sh") + 11; + let end = start + text[start..].find(')').unwrap(); + let listed: Vec<&str> = text[start..end].split_whitespace().collect(); + assert!(listed.len() > 10, "{listed:?}"); + assert_eq!(listed, crate::rerun::SWITCHES); } #[test] From 85e7455b0a6239e458421d0e8cc44bcd9e08b359 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:08:33 +0100 Subject: [PATCH 85/86] fix(ci): fail when the PCGR image or bundle moves without its measured size and digest The PCGR bundle is pinned by size and SHA-256, which the server does not publish. PCGR_DATA_MEASURED_FOR names the image tag and bundle date they were measured for, and check-coupled-versions.sh fails when either moves alone, when the size or digest is missing, or when either is malformed. The Renovate rule now says why it never moves the bundle. --- docs/00-reference-setup.md | 1 + renovate.json | 2 +- scripts/ci/check-coupled-versions.sh | 37 ++++++++++++++++++++++------ versions.env | 7 +++++- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index fa2236f7d..b6baa03d2 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -201,6 +201,7 @@ Setup installs it by default. To add the clinical data to an existing setup: ``` The installer checks the archive against `PCGR_DATA_SHA256` in `versions.env`, then extracts it into a temporary folder. +The server publishes no checksum, so the size (`PCGR_DATA_BYTES`) and the digest are measured from a full download. `PCGR_DATA_MEASURED_FOR` names the image tag and bundle date they belong to, and `scripts/ci/check-coupled-versions.sh` fails when `PCGR_IMAGE` or `PCGR_DATA_BUNDLE` moves without it. Only a complete tree is moved to `pcgr_data/20260620/data/`. The archive is deleted after installation. The [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) names the bundle's third-party databases. Their own terms apply. Setup prints the resource link before downloading. diff --git a/renovate.json b/renovate.json index 14f063c4e..09ad6ba29 100644 --- a/renovate.json +++ b/renovate.json @@ -187,7 +187,7 @@ "versioning": "regex:^release_(?\\d+)\\.(?\\d+)$" }, { - "description": "PCGR/CPSR: every update waits for approval on the dependency dashboard, because each release reads its own reference bundle and VEP cache (PCGR_DATA_BUNDLE and PCGR_VEP_CACHE_RELEASE in versions.env). The image comes from ghcr.io/sigven/pcgr (Docker Hub's sigven/pcgr stops at 2.2.5); the image matcher reads ghcr.io like any other registry. allowedVersions keeps the release tags (x.y.z): ghcr also carries development builds (2.3.2.9006) and .singularity tags.", + "description": "PCGR/CPSR: every update waits for approval on the dependency dashboard, because each release reads its own reference bundle and VEP cache (PCGR_DATA_BUNDLE and PCGR_VEP_CACHE_RELEASE in versions.env). Renovate never moves the bundle: no manager matches PCGR_DATA_BUNDLE, PCGR_DATA_BYTES or PCGR_DATA_SHA256, because the server publishes no checksum and the digest has to be measured from a full download. So an approved image PR fails scripts/ci/check-coupled-versions.sh until a person moves the bundle date, the VEP cache release, the size, the digest and PCGR_DATA_MEASURED_FOR in the same PR. The image comes from ghcr.io/sigven/pcgr (Docker Hub's sigven/pcgr stops at 2.2.5); the image matcher reads ghcr.io like any other registry. allowedVersions keeps the release tags (x.y.z): ghcr also carries development builds (2.3.2.9006) and .singularity tags.", "matchDatasources": ["docker"], "matchPackageNames": ["ghcr.io/sigven/pcgr"], "allowedVersions": "/^\\d+\\.\\d+\\.\\d+$/", diff --git a/scripts/ci/check-coupled-versions.sh b/scripts/ci/check-coupled-versions.sh index a00a76208..a50f1a8d9 100755 --- a/scripts/ci/check-coupled-versions.sh +++ b/scripts/ci/check-coupled-versions.sh @@ -7,7 +7,12 @@ # VEP_CACHE_RELEASE must equal the major of VEP_IMAGE (release_116.0 -> 116) # PYPGX_BUNDLE_VERSION must equal the version of PYPGX_IMAGE (0.26.0--pyh... -> 0.26.0) # ANNOTSV_ANNOTATIONS_VERSION must equal major.minor of ANNOTSV_IMAGE (3.5.10--h... -> 3.5) -# PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE and PCGR_DATA_BUNDLE must all be set +# PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE, PCGR_DATA_BUNDLE, PCGR_DATA_BYTES and +# PCGR_DATA_SHA256 must all be set, the size a byte +# count and the digest 64 hex digits +# PCGR_DATA_MEASURED_FOR must be " " +# (2.3.2 20260620): the size and digest were measured +# for that image and bundle, so either moving alone fails # YLEAF_DATA_VERSION must equal the version of YLEAF_IMAGE (3.2.1--pyh... -> 3.2.1) # GENCODE_RELEASE (scripts/lib/common.sh) must be the GENCODE release of the # Ensembl release of VEP_IMAGE, its major minus 66 (116 -> 50) @@ -31,7 +36,7 @@ check() { # shellcheck disable=SC2016 # $1 and ${!v} expand in the inner shell vals=$(env -i bash --noprofile --norc -c '. "$1"; for v in VEP_IMAGE VEP_CACHE_RELEASE PYPGX_IMAGE PYPGX_BUNDLE_VERSION \ ANNOTSV_IMAGE ANNOTSV_ANNOTATIONS_VERSION PCGR_IMAGE PCGR_VEP_CACHE_RELEASE PCGR_DATA_BUNDLE \ - YLEAF_IMAGE YLEAF_DATA_VERSION VERIFYBAMID2_IMAGE; do + PCGR_DATA_BYTES PCGR_DATA_SHA256 PCGR_DATA_MEASURED_FOR YLEAF_IMAGE YLEAF_DATA_VERSION VERIFYBAMID2_IMAGE; do printf "%s=%s\n" "$v" "${!v-}"; done' _ "$1") || { echo "FAIL: cannot read $1"; return 1; } get() { sed -n "s/^$1=//p" <<<"$vals"; } tag() { local i; i=$(get "$1"); i=${i##*/}; [[ "$i" == *:* ]] && echo "${i#*:}"; } @@ -64,15 +69,24 @@ check() { t=$(tag VERIFYBAMID2_IMAGE || true); t=${t%%--*} rule "VB2_PANEL_FILES (scripts/setup.sh) come from the VerifyBamID tag of VERIFYBAMID2_IMAGE" "$tags" "${t:+v$t}" local v missing="" - for v in PCGR_IMAGE PCGR_VEP_CACHE_RELEASE PCGR_DATA_BUNDLE; do + for v in PCGR_IMAGE PCGR_VEP_CACHE_RELEASE PCGR_DATA_BUNDLE PCGR_DATA_BYTES PCGR_DATA_SHA256; do [ -n "$(get "$v")" ] || missing="${missing} ${v}" done if [ -z "$missing" ]; then - echo "OK PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE and PCGR_DATA_BUNDLE are set together" + echo "OK PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE, PCGR_DATA_BUNDLE, PCGR_DATA_BYTES and PCGR_DATA_SHA256 are set together" else echo "FAIL PCGR variables must be set together; unset:${missing}" fail=1 fi + if [[ "$(get PCGR_DATA_BYTES)" =~ ^[1-9][0-9]*$ ]] && [[ "$(get PCGR_DATA_SHA256)" =~ ^[0-9a-f]{64}$ ]]; then + echo "OK PCGR_DATA_BYTES is a byte count and PCGR_DATA_SHA256 a SHA-256 digest" + else + echo "FAIL PCGR_DATA_BYTES must be a byte count and PCGR_DATA_SHA256 64 lowercase hex digits: $(get PCGR_DATA_BYTES), $(get PCGR_DATA_SHA256)" + fail=1 + fi + t=$(tag PCGR_IMAGE || true) + rule "PCGR_DATA_MEASURED_FOR names the PCGR_IMAGE tag and PCGR_DATA_BUNDLE of PCGR_DATA_BYTES and PCGR_DATA_SHA256" \ + "$(get PCGR_DATA_MEASURED_FOR)" "$( [ -n "$t" ] && [ -n "$(get PCGR_DATA_BUNDLE)" ] && echo "$t $(get PCGR_DATA_BUNDLE)")" return "$fail" } @@ -103,20 +117,29 @@ self_test() { [ "$rc" -eq 0 ] || { echo "self-test: versions.env itself failed:"; printf '%s\n' "$out"; fail=1; } # The values the unchanged file holds, so a legitimate bump keeps this # test green; each planted value differs from them. - local vep pypgx annotsv yleaf vb2 + local vep pypgx annotsv yleaf vb2 pcgr vep=$(sed -nE 's/^VEP_IMAGE="[^"]*:release_([0-9]+).*/\1/p' "${ROOT}/versions.env") pypgx=$(sed -nE 's/^PYPGX_IMAGE="[^"]*:([^"]*)--.*/\1/p' "${ROOT}/versions.env") annotsv=$(sed -nE 's/^ANNOTSV_IMAGE="[^"]*:([0-9]+\.[0-9]+).*/\1/p' "${ROOT}/versions.env") yleaf=$(sed -nE 's/^YLEAF_IMAGE="[^"]*:([^"]*)--.*/\1/p' "${ROOT}/versions.env") vb2=$(sed -nE 's/^VERIFYBAMID2_IMAGE="[^"]*:([^"]*)--.*/\1/p' "${ROOT}/versions.env") - if [ -z "$vep" ] || [ -z "$pypgx" ] || [ -z "$annotsv" ] || [ -z "$yleaf" ] || [ -z "$vb2" ]; then - echo "self-test: cannot read VEP_IMAGE, PYPGX_IMAGE, ANNOTSV_IMAGE, YLEAF_IMAGE or VERIFYBAMID2_IMAGE from versions.env"; return 1 + pcgr=$(sed -nE 's/^PCGR_IMAGE="[^"]*:([^"]*)".*/\1/p' "${ROOT}/versions.env") + if [ -z "$vep" ] || [ -z "$pypgx" ] || [ -z "$annotsv" ] || [ -z "$yleaf" ] || [ -z "$vb2" ] || [ -z "$pcgr" ]; then + echo "self-test: cannot read VEP_IMAGE, PYPGX_IMAGE, ANNOTSV_IMAGE, YLEAF_IMAGE, VERIFYBAMID2_IMAGE or PCGR_IMAGE from versions.env"; return 1 fi expect vep-cache "s/^VEP_CACHE_RELEASE=.*/VEP_CACHE_RELEASE=\"$((vep - 1))\"/" "^FAIL VEP_CACHE_RELEASE .*: $((vep - 1)), want ${vep}\$" expect vep-image "s/^(VEP_IMAGE=\"[^\"]*:release_)[0-9]+/\\1$((vep + 1))/" "^FAIL VEP_CACHE_RELEASE .*: ${vep}, want $((vep + 1))\$" expect pypgx 's/^PYPGX_BUNDLE_VERSION=.*/PYPGX_BUNDLE_VERSION="9.9.9"/' "^FAIL PYPGX_BUNDLE_VERSION .*: 9\\.9\\.9, want ${pypgx//./\\.}\$" expect annotsv 's/^ANNOTSV_ANNOTATIONS_VERSION=.*/ANNOTSV_ANNOTATIONS_VERSION="0.1"/' "^FAIL ANNOTSV_ANNOTATIONS_VERSION .*: 0\\.1, want ${annotsv//./\\.}\$" expect pcgr '/^PCGR_DATA_BUNDLE=/d' '^FAIL PCGR variables must be set together; unset: PCGR_DATA_BUNDLE' + expect pcgr-sha256 '/^PCGR_DATA_SHA256=/d' '^FAIL PCGR variables must be set together; unset: PCGR_DATA_SHA256' + expect pcgr-bytes 's/^PCGR_DATA_BYTES=.*/PCGR_DATA_BYTES="7.6 GB"/' '^FAIL PCGR_DATA_BYTES must be a byte count' + expect pcgr-digest 's/^PCGR_DATA_SHA256=.*/PCGR_DATA_SHA256="0123"/' '^FAIL PCGR_DATA_BYTES must be a byte count and PCGR_DATA_SHA256 64' + # A new bundle date, or a new image, without a new size and digest. + expect pcgr-bundle-alone 's/^PCGR_DATA_BUNDLE=.*/PCGR_DATA_BUNDLE="29991231"/' \ + "^FAIL PCGR_DATA_MEASURED_FOR .*: ${pcgr//./\\.} [0-9]+, want ${pcgr//./\\.} 29991231\$" + expect pcgr-image-alone 's/^(PCGR_IMAGE="[^"]*:)[^"]*/\19.9.9/' \ + "^FAIL PCGR_DATA_MEASURED_FOR .*: ${pcgr//./\\.} [0-9]+, want 9\\.9\\.9 [0-9]+\$" expect yleaf 's/^YLEAF_DATA_VERSION=.*/YLEAF_DATA_VERSION="9.9.9"/' "^FAIL YLEAF_DATA_VERSION .*: 9\\.9\\.9, want ${yleaf//./\\.}\$" expect gencode "s/^(GENCODE_RELEASE=)[^ ]*/\\1$((vep - 67))/" "^FAIL GENCODE_RELEASE .*: $((vep - 67)), want $((vep - 66))\$" common expect gencode-vep "s/^(VEP_IMAGE=\"[^\"]*:release_)[0-9]+/\\1$((vep + 1))/" "^FAIL GENCODE_RELEASE .*: $((vep - 66)), want $((vep - 65))\$" diff --git a/versions.env b/versions.env index 5c9595f9d..c305cb838 100644 --- a/versions.env +++ b/versions.env @@ -190,7 +190,12 @@ VEP_CACHE_RELEASE="116" # with PCGR_IMAGE. PCGR_VEP_CACHE_RELEASE="115" PCGR_DATA_BUNDLE="20260620" -# Exact archive size from the server; SHA-256 is measured from the full download. +# Exact archive size from the server; SHA-256 is measured from the full download +# (the server publishes no checksum). PCGR_DATA_MEASURED_FOR names the PCGR_IMAGE +# tag and PCGR_DATA_BUNDLE they were measured for: scripts/ci/check-coupled-versions.sh +# fails when the image or the bundle moves without it, so a bump downloads the +# new bundle and records its size and digest in the same change. +PCGR_DATA_MEASURED_FOR="2.3.2 20260620" PCGR_DATA_BYTES="7664082467" PCGR_DATA_SHA256="f9b8149e37c7e63bd7d877713cdb11a1a609c492b3f1a055fb7e90cd80273e54" # pypgx-bundle git tag (step 32): must equal the version of PYPGX_IMAGE. From c26a02a840663743ce755bc4a61092ea7852e6bb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:04:31 +0100 Subject: [PATCH 86/86] docs(updating): name the PCGR size, digest and measured-for pins in the bump steps --- docs/updating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/updating.md b/docs/updating.md index 17f2561f9..a182c8396 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -76,7 +76,7 @@ Each hold is a rule in `renovate.json` with its reason in the rule's `descriptio - **PharmCAT**: diff the diplotype table on the HG002 fixture between the old and the new image, and run the CPIC parser tests on a `report.json` from the new version. - **VEP major**: move `VEP_CACHE_RELEASE` to the new major and `GENCODE_RELEASE` (`scripts/lib/common.sh`) to the new major minus 66 in the same PR, download the new cache (about 26 GB) and rerun step 13. CI cannot run the offline cache. -- **PCGR/CPSR**: move `PCGR_DATA_BUNDLE` and `PCGR_VEP_CACHE_RELEASE` with the image, and rerun step 17 on a sample with an earlier result. CI does not run CPSR on a sample. The image test checks that `cpsr --help` still lists every flag step 17 passes, that `cpsr --version` names the release line of the image tag and the `pcgrr` R package its full release, and runs the bundled VEP with `--database` on 50 variants. +- **PCGR/CPSR**: move `PCGR_DATA_BUNDLE` and `PCGR_VEP_CACHE_RELEASE` with the image. Download the new bundle once, measure its size and sha256, and set `PCGR_DATA_BYTES`, `PCGR_DATA_SHA256` and `PCGR_DATA_MEASURED_FOR` (` `) in the same PR: the server publishes no checksum, and `scripts/ci/check-coupled-versions.sh` fails until all of them move together. Then rerun step 17 on a sample with an earlier result. CI does not run CPSR on a sample. The image test checks that `cpsr --help` still lists every flag step 17 passes, that `cpsr --version` names the release line of the image tag and the `pcgrr` R package its full release, and runs the bundled VEP with `--database` on 50 variants. - **Python**: resolve `scripts/cyrius-constraints.txt` again in the new image, as its header says, and rerun step 21 on a test BAM. - **Nextflow**: the PR runs the stub and E2E jobs on the new version. The prose names `NEXTFLOW_VERSION` rather than a number, and the fake nextflow of the fake-docker suite reads it from `versions.env`, so a patch needs no hand edit. A new release line (26.10 to 27.04, say) also needs the line named in `nextflow.config`, `nextflow.yml` and the prose of [Nextflow Execution](nextflow.md) and [Lessons Learned](lessons-learned.md) moved by hand. - **PGS scoring files and old PRS summaries**: after a scoring file update, rerun step 25 and treat its output as a new baseline ([PRS](25-prs.md#maintenance)). A `_prs_summary.tsv` whose header has `Variants_Used` comes from the scoring path used before 2026-10-01: its `Score` is plink2's average, its `Variants_Used` a dosage sum, and its PGS000738 row (a vitiligo score) is labelled schizophrenia. Rerun step 25 rather than reuse or compare it ([Lessons Learned](lessons-learned.md#a-prs-summary-with-a-variants_used-column-comes-from-the-old-scoring-path)).
{half}LOCUS_B20/75full_mutationraw score only
chr", page)) >= 50) - # 9. run_delta with the additive field + # 9. og run delta with the additive field prev = copy.deepcopy(s) del prev["sections"]["mito"]["data"]["heteroplasmic_sites"] - check("9. run_delta: no change for the added heteroplasmic_sites", run_delta.compare(prev, {}, s, {}) == [], - run_delta.compare(prev, {}, s, {})) + got = run_delta(prev, s) + check("9. og run delta: no change for the added heteroplasmic_sites", got == [], got) moved = copy.deepcopy(s) moved["sections"]["clinvar"]["data"]["hits"][0]["significance"] = "Benign" - check("9. red control: a reclassified hit is a change", run_delta.compare(prev, {}, moved, {}) != []) + check("9. red control: a reclassified hit is a change", run_delta(prev, moved) not in ([], None)) + + +def run_delta(prev, cur): + """The changes og run delta lists between two summaries (no manifests), or None if it failed.""" + with tempfile.TemporaryDirectory() as work: + os.makedirs(os.path.join(work, "prev")) + for path, summ in ((os.path.join(work, "prev", "summary.json"), prev), (os.path.join(work, "cur.json"), cur)): + with open(path, "w") as f: + json.dump(summ, f) + out = os.path.join(work, "delta") + p = subprocess.run([OG, "run", "delta", "--sample", "S", "--summary", os.path.join(work, "cur.json"), + "--previous", os.path.join(work, "prev"), "--out", out], capture_output=True, text=True) + if p.returncode: + print(p.stderr) + return None + with open(out + ".json") as f: + return json.load(f)["changes"] def page_problems(b, base, run): diff --git a/tests/test_run_state.py b/tests/test_run_state.py index dd4bf83a3..03feaec3d 100755 --- a/tests/test_run_state.py +++ b/tests/test_run_state.py @@ -233,7 +233,7 @@ def catalogue_checks(steps, sections, run_all_set): out["the recorded steps are what run-all.sh records"] = (recorded == want, sorted(recorded ^ want)) out["RUN_ALL_STEPS is the recorded steps"] = (set(run_all_set) == recorded, sorted(set(run_all_set) ^ recorded)) owner = {k: s["step"] for s in steps for k in s["sections"]} - keys = {k: step for k, _, step, _ in sections} + keys = {k: step for k, _, step in sections} wrong = [(k, step, owner.get(k)) for k, step in keys.items() if owner.get(k) != step] out["every SECTIONS key belongs to the row of its step"] = (not wrong and set(owner) <= set(keys), (wrong, sorted(set(owner) - set(keys)))) diff --git a/tests/test_str_ranges.py b/tests/test_str_ranges.py index 8c34556a1..6482d4152 100644 --- a/tests/test_str_ranges.py +++ b/tests/test_str_ranges.py @@ -9,14 +9,14 @@ no_thresholds; intermediate_min and intermediate_max are STRchive's own. Controls: a table with one row dropped, or one row twice, is caught. 2. bin/str_intermediate_ranges.tsv, the GeneReviews ranges the report names - (HTT, FMR1, DMPK): the columns bin/collect_summary.py reads, one row per + (HTT, FMR1, DMPK): the columns og summary collect reads, one row per locus, whole numbers with min <= max ending right below the next class, a label and a GeneReviews citation; docs/interpreting-results.md prints the same range in the locus's row; every locus has a row in str_ranges.tsv, and the range lies above its normal_max and below its pathogenic_min, so a count the report calls intermediate is one Stranger calls pre_mutation - (control: a planted HTT range of 20-35 is caught); collect_summary reads - every row. + (control: a planted HTT range of 20-35 is caught). That og reads every + row is og-summary's own test (collect::group_c::tests). 3. The "Key Disease Thresholds" table of docs/09-str-expansions.md: each row's normal maximum, pre-mutation band and full-mutation minimum are the catalog's (control: the table with HTT's minimum changed to 37 is caught). @@ -31,7 +31,6 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) sys.path.insert(0, os.path.join(REPO, "tests")) -import collect_summary # noqa: E402 import test_str_catalog # noqa: E402 TSV = os.path.join(REPO, "bin", "str_intermediate_ranges.tsv") @@ -130,7 +129,7 @@ def part_generated(cat, cat_path): def part_subset(gen): head, body = table(TSV) - check("the header is the columns collect_summary.py reads", head == COLUMNS, head) + check("the header is the columns og summary collect reads", head == COLUMNS, head) check("at least HTT, FMR1 and DMPK are listed", {"HTT", "FMR1", "DMPK"} <= {r[0] for r in body}, [r[0] for r in body]) loci = [r[0] for r in body] @@ -162,8 +161,6 @@ def inside(locus, lo, hi): f"({g.get('normal_max')}) and below its pathogenic minimum ({g.get('pathogenic_min')})", inside(d["locus"], lo, hi), g) check("control: a planted HTT range of 20-35 is outside STRchive's pre-mutation band", not inside("HTT", 20, 35)) - got = collect_summary.str_ranges(TSV) - check("collect_summary.str_ranges reads every row", sorted(got) == sorted(loci), sorted(got)) def threshold_rows(text): diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index cf55a9599..07fb6de3b 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -13,7 +13,6 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import traits_report as tr # noqa: E402 TABLE = os.path.join(REPO, "assets", "traits.tsv") RULES = os.path.join(REPO, "assets", "genotype_rules.tsv") CHECK = os.path.join(REPO, "scripts", "ci", "check-rule-alleles.sh") @@ -21,6 +20,36 @@ SMN_HEADER = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n" SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." FAILS = 0 +# The traits table's columns (og traits table --out). +OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", + "note", "pmids"] +# The copy-aware markers, pinned here as the test's own copy: og cn +# marker-regions must print the same copies (rust/crates/og-cn/src/markers.rs). +MARKERS = { + # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice + # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds + # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of + # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and + # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, + # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and + # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. + "LPA_KIV2_4925": { + "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", + "dbsnp": ("rs1484919544", "chr6", 160640667, "C", "T"), + "copies": (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932), + "context": "GACTCCTTACCTTGTTCGGAA", + "min_depth": 100, "min_alt": 5, "min_fraction": 0.008, "max_absent": 2, + }, + # SMN1 g.27134T>G is NM_000344.4:c.*3+80T>G in intron 7 (plus strand). + # SMN2 holds the same 21 bases 875,420 bases upstream, as at exon 7. + "SMN1_27134": { + "name": "SMN1 g.27134T>G", "chrom": "chr5", "ref": "T", "alt": "G", + "dbsnp": ("rs143838139", "chr5", 70952074, "T", "G"), + "copies": (70952074, 70076654), + "context": "TTTTTGAACATTTAAAAAGTT", + "min_depth": 10, "min_alt": 3, "min_fraction": 0.05, "max_absent": 1, + }, +} APOE_GOLDEN = { ("T/T", "C/C"): "e3d7512b649f1d23f5acbcaa08da19bd6a009d5b39ac57cb6856591c50e7cfda", ("T/T", "C/T"): "62e910d8ec7ac45ab15e9c9ed85c93b128ac41703d1149a332082dc0d7b5f389", @@ -120,7 +149,7 @@ def marker_errors(name, m=None, dbsnp=None): is one of its copies with its ref and alt, on the plus strand and not swapped; every copy sits in the middle of the same context, whose middle base is the ref.""" - m = tr.MARKERS[name] if m is None else m + m = MARKERS[name] if m is None else m rs, chrom, pos, ref, alt = m["dbsnp"] if dbsnp is None else dbsnp comp = {"A": "T", "C": "G", "G": "C", "T": "A"} errs = [] @@ -143,7 +172,7 @@ def marker_errors(name, m=None, dbsnp=None): def marker_checks(): """11. The copy-aware markers.""" - for name, m in tr.MARKERS.items(): + for name, m in MARKERS.items(): check(f"marker {name}: the pinned dbSNP record agrees with the marker", marker_errors(name) == [], marker_errors(name)) rs, chrom, pos, ref, alt = m["dbsnp"] @@ -159,16 +188,16 @@ def marker_checks(): e = marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) check(f"marker {name} control: a context whose middle base is not the ref is caught", any("middle base" in x for x in e), e) - check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(tr.MARKERS["LPA_KIV2_4925"]["copies"]) == 7) + check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(MARKERS["LPA_KIV2_4925"]["copies"]) == 7) if not OG: print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") return - for name, m in tr.MARKERS.items(): + for name, m in MARKERS.items(): res = og_cn("marker-regions", name) check(f"marker {name}: og cn marker-regions prints one region per copy of MARKERS", res.returncode == 0 and res.stdout.split() == [f"{m['chrom']}:{p}-{p}" for p in m["copies"]], res) - sm = tr.MARKERS["SMN1_27134"] + sm = MARKERS["SMN1_27134"] work = tempfile.mkdtemp() try: def pileup(t_reads, g_reads, extra=""): @@ -305,11 +334,8 @@ def plain_tsv_checks(): check(f"plain TSV control: {name} is refused", res.returncode == 1, res.stderr) check(f"plain TSV control: {name} preserves both outputs", previous == (open(out, "rb").read(), open(rout, "rb").read())) - try: - _, collected = cs.sec_traits(work, "S") - readable = bool(collected["rows"]) - except cs.Unreadable: - readable = False + sec = cs.collect("S", work)["sections"]["traits"] + readable = sec["state"] == "ok" and bool(sec["data"]["rows"]) check(f"plain TSV control: {name} leaves collector-readable output", readable) @@ -348,7 +374,7 @@ def table(records): [("44908684", "T", "C", a), ("44908822", "C", "T", b)] if gt != "."] traits, lines = table(records) apoe = next(r for r in traits if "+" in r["rsid"]) - got = hashlib.sha256("\t".join(apoe[k] for k in tr.OUT_COLUMNS).encode()).hexdigest() + got = hashlib.sha256("\t".join(apoe[k] for k in OUT_COLUMNS).encode()).hexdigest() check(f"APOE golden {a}, {b}", got == want, got) for rid, cases in MULTI_SNP.items(): for gts, label in cases: @@ -378,7 +404,7 @@ def table(records): # Collect and render the actual CLI output, independently for each rule. records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(sites, gts)] _, lines = table(records) - read = cs.genotype_rules(work, "S") or [] + read = cs.collect("S", work)["sections"]["traits"]["data"]["rules"] or [] page = "\n".join(rr.rules_html({"rules": read}, "S")) got = next(r for r in lines if r["rule"] == rid) check(f"HTML report: {rid} with result {want}", rr.E(got["trait"]) in page and rr.E(want) in page, page[:300]) diff --git a/tests/test_traits_table.py b/tests/test_traits_table.py index 3d2566b27..984d426f1 100644 --- a/tests/test_traits_table.py +++ b/tests/test_traits_table.py @@ -7,6 +7,7 @@ positive, wrong-base and missing-contig controls without network access. """ import argparse +import csv import json import os import shutil @@ -16,9 +17,6 @@ import time REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import traits_report as tr # noqa: E402 - TABLE = os.path.join(REPO, "assets", "traits.tsv") FAILS = 0 NC = {f"NC_0000{n:02d}": f"chr{n}" for n in range(1, 23)} @@ -122,13 +120,20 @@ def online_errors(rows): return errs +def read_table(path): + """The rows of a traits table as dicts (its schema is og traits check's to judge).""" + with open(path, encoding="utf-8") as f: + return list(csv.DictReader((line for line in f if line.strip() and not line.startswith("#")), + delimiter="\t", quoting=csv.QUOTE_NONE)) + + def schema(path): return subprocess.run([os.environ.get("OG", os.path.join(REPO, "bin", "og")), "traits", "check", "--table", path], capture_output=True, text=True) def schema_controls(): - good = tr.read_table(TABLE) + good = read_table(TABLE) check("the table has at least 30 rows", len(good) >= 30, len(good)) work = tempfile.mkdtemp() try: @@ -174,13 +179,9 @@ def main(argv=None): return 0 schema_controls() else: - try: - rows = tr.read_table(a.table) - except tr.TableError as e: - check("the table can be read", False, e) - rows = [] res = schema(a.table) check(f"{a.table} passes the schema check", res.returncode == 0, res.stderr) + rows = read_table(a.table) if res.returncode == 0 else [] if a.fasta and rows: e, n = fasta_errors(a.fasta, rows) print(f" ref checked against {a.fasta}: {n} rows (the others are on contigs it lacks)") diff --git a/tests/test_viral_report.py b/tests/test_viral_report.py index 96d14505d..7d16316a9 100644 --- a/tests/test_viral_report.py +++ b/tests/test_viral_report.py @@ -206,6 +206,20 @@ def drop_genome(d): import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + + def og_report(d): """og viral report on the inputs in d, into d/S/viral/S_viral.tsv.""" @@ -218,12 +232,12 @@ def og_report(d): write_inputs(d, {B: flat(15), EBV: flat(1), PHIX: flat(3)}, {B: 400, EBV: 30, PHIX: 20}) os.makedirs(f"{d}/S/viral") og_report(d) - path, data = cs.sec_viral(f"{d}/S", "S") + path, data = sec(f"{d}/S", "S", "viral") check("8 the summary finds the table", path.endswith("viral/S_viral.tsv"), True) check("8 the summary's HHV-6 line", data["hhv6"], "inherited integrated HHV-6B likely") check("8 the summary lists what was found", [x["virus"] for x in data["found"]], ["HHV-6B", "Human gammaherpesvirus 4", "Escherichia phage phiX174"]) - check("8 no table: no section", cs.sec_viral(f"{d}/none", "S"), (None, {})) + check("8 no table: no section", sec(f"{d}/none", "S", "viral"), (None, {})) lines = rr.viral_lines(data) check("8 the text block opens with the HHV-6 line", lines[0], "HHV-6: inherited integrated HHV-6B likely") check("8 and ends with the note", lines[-1], rr.VIRAL_NOTE) @@ -232,9 +246,9 @@ def og_report(d): with open(f"{d}/S/viral/S_viral.tsv", "w") as f: f.write("virus\tcall\n") try: - cs.sec_viral(f"{d}/S", "S") + sec(f"{d}/S", "S", "viral") check("8 a table without its columns is unreadable", "read", "Unreadable") - except cs.Unreadable: + except Unreadable: check("8 a table without its columns is unreadable", "Unreadable", "Unreadable") # 8. the summary's HHV-6 line without a human depth, and with one but no integration @@ -244,7 +258,7 @@ def og_report(d): write_inputs(d, {EBV: flat(1)}, {EBV: 30}, human=human) os.makedirs(f"{d}/S/viral") og_report(d) - data = cs.sec_viral(f"{d}/S", "S")[1] + data = sec(f"{d}/S", "S", "viral")[1] check(f"8 the summary's HHV-6 line reads {want!r}", data["hhv6"], want) if human: check("8 no human depth: the text line leaves out copies per cell", rr.viral_lines(data)[1], @@ -255,7 +269,7 @@ def og_report(d): write_inputs(d, {HCV: lambda i: 1 if i < 50 else 0}, {HCV: 5}) os.makedirs(f"{d}/S/viral") og_report(d) - data = cs.sec_viral(f"{d}/S", "S")[1] + data = sec(f"{d}/S", "S", "viral")[1] check("8 a genome with 5 reads is listed", [(x["virus"], x["call"], x["reads"]) for x in data["found"]], [("Hepatitis C virus genotype 1", CALL_NONE, "5")]) check("8 and the text does not say no read", any(ln.startswith("No read") for ln in rr.viral_lines(data)), False) diff --git a/tests/test_vntyper_report.py b/tests/test_vntyper_report.py index c4341a997..8c1ac8384 100644 --- a/tests/test_vntyper_report.py +++ b/tests/test_vntyper_report.py @@ -192,7 +192,8 @@ def sample(name, kestrel, **kw): s = collect_summary.collect("S", sd) return s, render_report.text_report(s), render_report.html_report(s) - line = collect_summary.VNTYPER_NOT_ASSESSED + line = ("MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; " + "no standard caller sees them") s, txt, html = sample("pos", call()) sec = s["sections"].get("vntyper", {}) check("6. the section is read", sec.get("state") == "ok" and sec["data"]["status"] == "positive", sec) diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py index 77c49ea30..64e4815df 100644 --- a/tests/test_ystr_report.py +++ b/tests/test_ystr_report.py @@ -305,19 +305,33 @@ def good(gb="0", dp="20"): import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + + sd = os.path.join(TMP, "sample") os.makedirs(os.path.join(sd, "y_str")) v = vcf({"DYS393": good("4"), "DYS390": ("0", "0", "0.6", "20", "0", "0")}) subprocess.run([OG, "y", "str-report", "--panel", PANEL, "--vcf", v, "--tsv", os.path.join(sd, "y_str", "S1_y_str.tsv"), "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")], check=True, capture_output=True) -path, data = cs.sec_y_str(sd, "S1") +path, data = sec(sd, "S1", "y_str") check("7 collect_summary finds the table", os.path.basename(path or ""), "S1_y_str.tsv") check("7 counts per confidence", (data["high"], data["low"], data["not_callable"], data["no_call"]), (1, 1, sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) + 3, # + DYS452, DYS389II, DYS448: too long 111 - 2 - sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) - 3)) check("7 the markers keep their order", [x["marker"] for x in data["markers"]], names) -check("7 a sample without the step has no section", cs.sec_y_str(TMP, "nobody"), (None, {})) +check("7 a sample without the step has no section", sec(TMP, "nobody", "y_str"), (None, {})) html = "\n".join(rr.ystr_body(data)) check("7 the card lists DYS393 at 13", "DYS39313highDYS39024low