From 52d4fd0cf12640303f92064ebc13f6cfc61e1a12 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:53:53 +0100 Subject: [PATCH 1/4] test(str): reproduce pathogenic motif minorities hidden by benign reads Bead: sv-28 --- .../og-bam/tests/i411_rfc1_motif_minority.rs | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs diff --git a/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs b/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs new file mode 100644 index 00000000..4ef83f39 --- /dev/null +++ b/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs @@ -0,0 +1,144 @@ +//! Synthetic BAM regressions for pathogenic and unknown motifs under a benign majority. +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use rust_htslib::bam::{self, header::HeaderRecord, record::{Cigar, CigarString}}; + +struct Scratch(PathBuf); +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn count(counts: &[(&str, usize)], dominant: bool) -> (String, String) { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let dir = Scratch(std::env::temp_dir().join(format!( + "og-i411-motif-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed) + ))); + fs::create_dir_all(&dir.0).unwrap(); + let bam_path = dir.0.join("reads.bam"); + let vcf = dir.0.join("calls.vcf"); + let out = dir.0.join("motif.tsv"); + let catalog = dir.0.join("catalog.json"); + let locus = if dominant { "FAMEX" } else { "RFC1" }; + fs::write(&vcf, format!( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\nchr4\t101\t.\tA\t\t.\tPASS\tEND=155;REPID={locus};STR_STATUS=full_mutation\n" + )).unwrap(); + fs::write(&catalog, r#"[{"LocusId":"FAMEX","ReferenceRegion":"chr4:100-155","Disease":"FAME","InheritanceMode":"AD","BenignMotifs":["TTTTA"],"PathogenicMotifs":["TTTCA"]}]"#).unwrap(); + let mut header = bam::Header::new(); + header.push_record(HeaderRecord::new(b"HD").push_tag(b"SO", "coordinate")); + header.push_record(HeaderRecord::new(b"SQ").push_tag(b"SN", "chr4").push_tag(b"LN", 1000)); + { + let mut writer = bam::Writer::from_path(&bam_path, &header, bam::Format::Bam).unwrap(); + for (motif, n) in counts { + for i in 0..*n { + let mut rec = bam::Record::new(); + // Both strands must keep the same evidence. 150 bases exceed the reference run. + let reverse = i % 2 == 1; + let seq: Vec = if reverse { + motif.bytes().rev().map(|c| match c { + b'A' => b'T', b'C' => b'G', b'G' => b'C', b'T' => b'A', _ => unreachable!() + }).collect::>().repeat(30) + } else { motif.as_bytes().repeat(30) }; + rec.set(format!("{motif}-{i}").as_bytes(), Some(&CigarString(vec![Cigar::Match(150)])), &seq, &[30; 150]); + rec.set_tid(0); + rec.set_pos(75); + rec.set_flags(if reverse { 16 } else { 0 }); + rec.set_mapq(60); + writer.write(&rec).unwrap(); + } + } + } + bam::index::build(&bam_path, None, bam::index::Type::Bai, 1).unwrap(); + let mut args = vec!["--bam".into(), bam_path.display().to_string(), "--vcf".into(), vcf.display().to_string(), "--out".into(), out.display().to_string()]; + if dominant { + args.extend(["--catalog".into(), catalog.display().to_string(), "--flagged".into()]); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + assert_eq!(og_bam::motif::cli(&args, &mut stdout, &mut stderr), 0, "{}", String::from_utf8_lossy(&stderr)); + (fs::read_to_string(out).unwrap(), String::from_utf8(stdout).unwrap()) +} + +fn row(table: &str) -> Vec<&str> { + table.lines().nth(1).unwrap().split('\t').collect() +} + +fn uncertain(counts: &[(&str, usize)], dominant: bool, evidence: &str) { + let (table, _) = count(counts, dominant); + let r = row(&table); + assert_eq!(r[9], "uncertain_motif", "{table}"); + assert!(r[10].contains(evidence), "{table}"); + assert!(r[10].contains("possible"), "{table}"); + assert!(!r[10].contains("the flag is not supported"), "{table}"); +} + +#[test] +fn rfc1_keeps_fifteen_percent_pathogenic_reads() { + uncertain(&[("AAAAG", 17), ("AAGGG", 3)], false, "AAGGG in 3 of 20"); +} + +#[test] +fn rfc1_keeps_one_or_two_pathogenic_reads_without_calling_them_pathogenic() { + uncertain(&[("AAAAG", 4), ("AAGGG", 2)], false, "AAGGG in 2 of 6"); + uncertain(&[("AAAAG", 19), ("AAGGG", 1)], false, "AAGGG in 1 of 20"); +} + +#[test] +fn dominant_insert_can_be_a_minority_and_still_needs_confirmation() { + uncertain(&[("TTTTA", 17), ("TTTCA", 3)], true, "TTTCA in 3 of 20"); + uncertain(&[("TTTTA", 999), ("TTTCA", 1)], true, "TTTCA in 1 of 1000"); +} + +#[test] +fn unknown_minority_and_tie_are_not_dismissed() { + // AGAGG is named AAGAG after strand/rotation normalization. + uncertain(&[("AAAAG", 6), ("AGAGG", 4)], false, "AAGAG in 4 of 10"); + uncertain(&[("AAAAG", 5), ("AGAGG", 5)], false, "AAGAG in 5 of 10"); + uncertain(&[("AAAAG", 19), ("AGAGG", 1)], false, "AAGAG in 1 of 20"); +} + +#[test] +fn twenty_and_twenty_five_percent_are_possible_without_resolving_both_copies() { + for counts in [[("AAAAG", 16), ("AAGGG", 4)], [("AAAAG", 15), ("AAGGG", 5)]] { + let (table, _) = count(&counts, false); + let r = row(&table); + assert_eq!(r[9], "pathogenic_motif"); + assert!(r[10].contains("confirm with repeat-primed PCR")); + assert!(r[10].contains("needs a pathogenic expansion on both copies")); + assert!(!r[10].contains("carrier")); + } +} + +#[test] +fn pure_and_mixed_known_benign_motifs_remain_benign() { + for counts in [vec![("AAAAG", 10)], vec![("AAAAG", 6), ("AAAGG", 4)]] { + let (table, _) = count(&counts, false); + assert_eq!(row(&table)[9], "benign_motif"); + } + let (table, _) = count(&[("TTTTA", 10)], true); + assert_eq!(row(&table)[9], "benign_motif"); +} + +#[test] +fn fewer_than_three_total_reads_remain_unresolved() { + let (table, _) = count(&[("AAGGG", 2)], false); + assert_eq!(row(&table)[9], "not_resolved"); +} + +#[test] +fn minority_outputs_match_the_deliberate_parity_expectations() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../tests/parity/want/rfc1"); + for (case, counts, dominant) in [ + ("pathogenic-minority", [("AAAAG", 17), ("AAGGG", 3)], false), + ("dominant-minority", [("TTTTA", 17), ("TTTCA", 3)], true), + ("unknown-tie", [("AAAAG", 5), ("AGAGG", 5)], false), + ] { + let (table, stdout) = count(&counts, dominant); + let want = fs::read_to_string(root.join(format!("{case}.tsv"))).unwrap(); + assert_eq!(table, want); + let r = row(&table); + assert_eq!(stdout, format!("{} motif count: {} ({})\n", r[0], r[9], r[10])); + } +} From 84b051cab90225174fb59a68e153ec4f81151dd0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:58:01 +0100 Subject: [PATCH 2/4] fix(str): retain pathogenic and unknown minority motif evidence Keep low-share evidence uncertain and show counts in the report note. Preserve the pathogenic threshold and avoid inferring phased alleles or carrier status from read shares. Bead: sv-28 --- CHANGELOG.md | 2 + docs/09b-stranger.md | 10 ++- rust/crates/og-bam/src/motif.rs | 40 ++++++++- .../og-bam/tests/i411_rfc1_motif_minority.rs | 82 +++++++++++++++---- tests/parity/want/rfc1/dominant-minority.tsv | 2 + .../parity/want/rfc1/pathogenic-minority.tsv | 2 + tests/parity/want/rfc1/unknown-tie.tsv | 2 + 7 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 tests/parity/want/rfc1/dominant-minority.tsv create mode 100644 tests/parity/want/rfc1/pathogenic-minority.tsv create mode 100644 tests/parity/want/rfc1/unknown-tie.tsv diff --git a/CHANGELOG.md b/CHANGELOG.md index d6409f33..8a84b194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The STR motif count no longer dismisses a flag when a benign majority hides pathogenic or unknown motif reads. It now reports `uncertain_motif` and names those reads and their shares, including a 50/50 tie. RFC1 still needs pathogenic expansions on both copies; the note makes clear that read counts do not resolve either copy. Dominant motif loci also keep minority evidence and explain that one pathogenic allele can be sufficient. The 3-read and 20% floors for `pathogenic_motif` stay unchanged. Output columns and verdict names stay unchanged. Part of #411 (sv-28). + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/09b-stranger.md b/docs/09b-stranger.md index bbd10dc7..89ba9428 100644 --- a/docs/09b-stranger.md +++ b/docs/09b-stranger.md @@ -80,16 +80,18 @@ About a second, measured in three 30-40x WGS runs ([runtime](hardware-requiremen | Verdict | When | The report says | |---|---|---| | `pathogenic_motif` | 3 or more reads, and at least 20% of them, carry AAGGG or ACAGG | confirm with repeat-primed PCR | - | `benign_motif` | otherwise, AAAAG or AAAGG carries at least half of the reads | the flag is not supported | - | `uncertain_motif` | otherwise, with 3 or more reads | the dominant motif is not a known benign one: confirm if symptoms fit | + | `benign_motif` | all counted reads carry known benign motifs, and AAAAG or AAAGG carries at least half | the flag is not supported by these reads | + | `uncertain_motif` | otherwise, with 3 or more reads in total, including a benign majority with any pathogenic or unknown motif reads | a pathogenic expansion remains possible, or the dominant motif is not known benign: confirm if symptoms fit | | `not_resolved` | fewer than 3 reads carry a run | the motif is not resolved | | `not_checked` | no BAM | confirm the flag with repeat-primed PCR | A rerun of the script keeps an existing table unless it is `not_checked`, so a BAM added later gets counted. + A benign majority cannot rule out a pathogenic expansion while pathogenic or unknown motif reads remain. For example, 17 AAAAG reads and 3 AAGGG reads give `uncertain_motif`. The note names AAGGG and its 15% share. One or two pathogenic reads among at least three total reads also keep the result uncertain. The 3-read and 20% floors still apply to `pathogenic_motif`. Neither verdict resolves the two alleles or establishes CANVAS. Even at 25% pathogenic reads, confirm the possible expansion with repeat-primed PCR. Benign and pathogenic motifs can come from one interrupted allele, so their shares do not establish a carrier state. An existing completed table is kept on rerun: remove it to recount with this rule. + In the Nextflow pipeline, `STRANGER_RFC1` runs the same count after `STRANGER`, in the same image, on the BAM of the samplesheet row (or the one the pipeline aligned), and writes `/expansion_hunter/_rfc1_motif.tsv` only when Stranger flags RFC1. `HTML_REPORT` gets the table, so the Nextflow report shows the same verdict as step 24's. - The count reads the motif; it does not size a long expansion, and a heterozygous AAGGG allele is a carrier state, not CANVAS. CI checks it on made-up reads (`tests/test_rfc1_motif.py`, the parity cases of `rust/crates/og-bam/tests/parity.rs`, `tests/e2e/expansions-and-rfc1-1-motif.sh`); how it does on real carriers, whose reads the aligner may clip or misplace differently, has not been measured. Confirm any `pathogenic_motif` or `uncertain_motif` with motif-aware, flanking-PCR or repeat-primed-PCR testing, and only if cerebellar-ataxia/neuropathy/vestibular symptoms are present. + The count reads the motif; it does not size a long expansion, and a heterozygous AAGGG allele is a carrier state, not CANVAS. CI checks it on made-up reads ([minority regressions](../rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs), `tests/test_rfc1_motif.py`, the parity cases of `rust/crates/og-bam/tests/parity.rs`, `tests/e2e/expansions-and-rfc1-1-motif.sh`); how it does on real carriers, whose reads the aligner may clip or misplace differently, has not been measured. Confirm any `pathogenic_motif` or `uncertain_motif` with motif-aware, flanking-PCR or repeat-primed-PCR testing, and only if cerebellar-ataxia/neuropathy/vestibular symptoms are present. With the STRchive catalog, the count takes RFC1's motifs and its two repeats (AAAAG and AAGGG, chr4:39348424-39348483) from the catalog: benign AAAAG and AAAGG, pathogenic AAGGG, ACAGG and CAGGG. STRchive lists AAAGG as pathogenic, and its own notes say why: expansions of 600 or more AAAGG units are seen in CANVAS, while 100 to 380 units are seen in unaffected people ([PMID 37450567](https://pubmed.ncbi.nlm.nih.gov/37450567/)). Short reads cannot size an allele that long, so the count keeps AAAGG benign, as [Cortese et al. 2019](https://pubmed.ncbi.nlm.nih.gov/30926972/) classed it. On the CI's made-up reads the verdicts are the same as with the built-in motifs. -- **The other loci where the motif decides.** At the FAME loci (familial adult myoclonic epilepsy: SAMD12, STARD7, MARCHF6, YEATS2, TNRC6A, RAPGEF2, RAI1; [Ishiura et al. 2018](https://pubmed.ncbi.nlm.nih.gov/29507423/)), SCA31 (BEAN1), SCA37 (DAB1) and aFTLD-U (GOLGA8A), disease comes from an inserted motif (TTTCA at the FAME loci), while an expansion of the reference motif is benign. The STRchive catalog names both classes of motif for each. When Stranger flags one of them, step 9b runs the same count (`og str motif --flagged`) with that locus's motifs and writes one row per flagged locus to `_str_motifs.tsv`, with the verdicts of the RFC1 table. A recessive locus's note asks for a pathogenic expansion on both copies; these are dominant. A rerun redoes a `not_checked` table, as for RFC1. `STRANGER_RFC1` in Nextflow writes the same file. Both reports show the verdict of each flagged locus in this table, under Repeat Expansions. CI checks it on made-up SAMD12 reads (`tests/e2e/str-catalog-1-strchive.sh`) and on a made-up catalog locus (the parity cases of `rust/crates/og-bam/tests/parity.rs`); how it does on real carriers has not been measured. +- **The other loci where the motif decides.** At the FAME loci (familial adult myoclonic epilepsy: SAMD12, STARD7, MARCHF6, YEATS2, TNRC6A, RAPGEF2, RAI1; [Ishiura et al. 2018](https://pubmed.ncbi.nlm.nih.gov/29507423/)), SCA31 (BEAN1), SCA37 (DAB1) and aFTLD-U (GOLGA8A), disease comes from an inserted motif (TTTCA at the FAME loci), while an expansion of the reference motif is benign. The STRchive catalog names both classes of motif for each. When Stranger flags one of them, step 9b runs the same count (`og str motif --flagged`) with that locus's motifs and writes one row per flagged locus to `_str_motifs.tsv`, with the verdicts of the RFC1 table. A recessive locus's note asks for a pathogenic expansion on both copies. At a dominant locus, one pathogenic allele can be sufficient. A pathogenic minority below the 3-read or 20% floor gives `uncertain_motif`, with its counts in the note, even when the reference motif dominates. A rerun redoes a `not_checked` table, as for RFC1. `STRANGER_RFC1` in Nextflow writes the same file. Both reports show the verdict of each flagged locus in this table, under Repeat Expansions. CI checks it on made-up SAMD12 reads (`tests/e2e/str-catalog-1-strchive.sh`) and on a made-up catalog locus (the parity cases of `rust/crates/og-bam/tests/parity.rs`); how it does on real carriers has not been measured. diff --git a/rust/crates/og-bam/src/motif.rs b/rust/crates/og-bam/src/motif.rs index 30cb0ddf..cc221768 100644 --- a/rust/crates/og-bam/src/motif.rs +++ b/rust/crates/og-bam/src/motif.rs @@ -20,9 +20,10 @@ //! [--catalog CATALOG.json [--flagged]] [--no-softclip] [--no-mates] //! ``` //! -//! The output is the Python's, byte for byte: a header and one row per +//! The output keeps the Python's columns: a header and one row per //! locus (`locus chrom start end repeat_reads dominant_motif dominant_class //! dominant_share motif_shares verdict note`). +//! A benign majority cannot dismiss pathogenic or unknown motif reads. use std::collections::{HashMap, HashSet}; use std::io::Write; @@ -437,7 +438,42 @@ fn verdict(counts: &IndexMap, units_min: i64, spec: &Spec) -> [St ), ); } - if cls == "benign" && share >= BENIGN_SHARE { + // Read shares cannot resolve alleles. In particular, a benign majority + // can coexist with a pathogenic insert at a dominant locus or with + // unresolved pathogenic evidence at a recessive locus. Keep even weak + // evidence uncertain, without promoting a few reads to pathogenic. + let unresolved: Vec = order + .iter() + .filter(|(m, c)| *c > 0 && spec.known.get(*m) != Some(&Class::Benign)) + .map(|(m, c)| { + format!( + "{m} in {c} of {total} repeat reads ({})", + percent(ratio(*c), 0) + ) + }) + .collect(); + if cls == "benign" && !unresolved.is_empty() { + let inheritance = if spec.recessive { + format!( + "{} needs a pathogenic expansion on both copies; these reads do not resolve either copy", + spec.disease + ) + } else { + "one pathogenic expansion can be sufficient; these reads do not resolve an allele" + .into() + }; + return row( + dom, + cls, + fixed(share, 2), + MotifVerdict::UncertainMotif, + format!( + "{}: a pathogenic expansion remains possible; {inheritance}; confirm with repeat-primed PCR if symptoms fit", + unresolved.join("; ") + ), + ); + } + if cls == "benign" && share >= BENIGN_SHARE && unresolved.is_empty() { return row( dom, cls, diff --git a/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs b/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs index 4ef83f39..95b614fe 100644 --- a/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs +++ b/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs @@ -3,7 +3,11 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; -use rust_htslib::bam::{self, header::HeaderRecord, record::{Cigar, CigarString}}; +use rust_htslib::bam::{ + self, + header::HeaderRecord, + record::{Cigar, CigarString}, +}; struct Scratch(PathBuf); impl Drop for Scratch { @@ -15,7 +19,9 @@ impl Drop for Scratch { fn count(counts: &[(&str, usize)], dominant: bool) -> (String, String) { static NEXT: AtomicUsize = AtomicUsize::new(0); let dir = Scratch(std::env::temp_dir().join(format!( - "og-i411-motif-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed) + "og-i411-motif-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) ))); fs::create_dir_all(&dir.0).unwrap(); let bam_path = dir.0.join("reads.bam"); @@ -29,7 +35,11 @@ fn count(counts: &[(&str, usize)], dominant: bool) -> (String, String) { fs::write(&catalog, r#"[{"LocusId":"FAMEX","ReferenceRegion":"chr4:100-155","Disease":"FAME","InheritanceMode":"AD","BenignMotifs":["TTTTA"],"PathogenicMotifs":["TTTCA"]}]"#).unwrap(); let mut header = bam::Header::new(); header.push_record(HeaderRecord::new(b"HD").push_tag(b"SO", "coordinate")); - header.push_record(HeaderRecord::new(b"SQ").push_tag(b"SN", "chr4").push_tag(b"LN", 1000)); + header.push_record( + HeaderRecord::new(b"SQ") + .push_tag(b"SN", "chr4") + .push_tag(b"LN", 1000), + ); { let mut writer = bam::Writer::from_path(&bam_path, &header, bam::Format::Bam).unwrap(); for (motif, n) in counts { @@ -38,11 +48,27 @@ fn count(counts: &[(&str, usize)], dominant: bool) -> (String, String) { // Both strands must keep the same evidence. 150 bases exceed the reference run. let reverse = i % 2 == 1; let seq: Vec = if reverse { - motif.bytes().rev().map(|c| match c { - b'A' => b'T', b'C' => b'G', b'G' => b'C', b'T' => b'A', _ => unreachable!() - }).collect::>().repeat(30) - } else { motif.as_bytes().repeat(30) }; - rec.set(format!("{motif}-{i}").as_bytes(), Some(&CigarString(vec![Cigar::Match(150)])), &seq, &[30; 150]); + motif + .bytes() + .rev() + .map(|c| match c { + b'A' => b'T', + b'C' => b'G', + b'G' => b'C', + b'T' => b'A', + _ => unreachable!(), + }) + .collect::>() + .repeat(30) + } else { + motif.as_bytes().repeat(30) + }; + rec.set( + format!("{motif}-{i}").as_bytes(), + Some(&CigarString(vec![Cigar::Match(150)])), + &seq, + &[30; 150], + ); rec.set_tid(0); rec.set_pos(75); rec.set_flags(if reverse { 16 } else { 0 }); @@ -52,13 +78,32 @@ fn count(counts: &[(&str, usize)], dominant: bool) -> (String, String) { } } bam::index::build(&bam_path, None, bam::index::Type::Bai, 1).unwrap(); - let mut args = vec!["--bam".into(), bam_path.display().to_string(), "--vcf".into(), vcf.display().to_string(), "--out".into(), out.display().to_string()]; + let mut args = vec![ + "--bam".into(), + bam_path.display().to_string(), + "--vcf".into(), + vcf.display().to_string(), + "--out".into(), + out.display().to_string(), + ]; if dominant { - args.extend(["--catalog".into(), catalog.display().to_string(), "--flagged".into()]); + args.extend([ + "--catalog".into(), + catalog.display().to_string(), + "--flagged".into(), + ]); } let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - assert_eq!(og_bam::motif::cli(&args, &mut stdout, &mut stderr), 0, "{}", String::from_utf8_lossy(&stderr)); - (fs::read_to_string(out).unwrap(), String::from_utf8(stdout).unwrap()) + assert_eq!( + og_bam::motif::cli(&args, &mut stdout, &mut stderr), + 0, + "{}", + String::from_utf8_lossy(&stderr) + ); + ( + fs::read_to_string(out).unwrap(), + String::from_utf8(stdout).unwrap(), + ) } fn row(table: &str) -> Vec<&str> { @@ -93,10 +138,10 @@ fn dominant_insert_can_be_a_minority_and_still_needs_confirmation() { #[test] fn unknown_minority_and_tie_are_not_dismissed() { - // AGAGG is named AAGAG after strand/rotation normalization. - uncertain(&[("AAAAG", 6), ("AGAGG", 4)], false, "AAGAG in 4 of 10"); - uncertain(&[("AAAAG", 5), ("AGAGG", 5)], false, "AAGAG in 5 of 10"); - uncertain(&[("AAAAG", 19), ("AGAGG", 1)], false, "AAGAG in 1 of 20"); + // AGAGG is already the smallest rotation on either strand. + uncertain(&[("AAAAG", 6), ("AGAGG", 4)], false, "AGAGG in 4 of 10"); + uncertain(&[("AAAAG", 5), ("AGAGG", 5)], false, "AGAGG in 5 of 10"); + uncertain(&[("AAAAG", 19), ("AGAGG", 1)], false, "AGAGG in 1 of 20"); } #[test] @@ -139,6 +184,9 @@ fn minority_outputs_match_the_deliberate_parity_expectations() { let want = fs::read_to_string(root.join(format!("{case}.tsv"))).unwrap(); assert_eq!(table, want); let r = row(&table); - assert_eq!(stdout, format!("{} motif count: {} ({})\n", r[0], r[9], r[10])); + assert_eq!( + stdout, + format!("{} motif count: {} ({})\n", r[0], r[9], r[10]) + ); } } diff --git a/tests/parity/want/rfc1/dominant-minority.tsv b/tests/parity/want/rfc1/dominant-minority.tsv new file mode 100644 index 00000000..aac3ac6d --- /dev/null +++ b/tests/parity/want/rfc1/dominant-minority.tsv @@ -0,0 +1,2 @@ +locus chrom start end repeat_reads dominant_motif dominant_class dominant_share motif_shares verdict note +FAMEX chr4 101 155 20 TTTTA benign 0.85 TTTTA=0.85,TTTCA=0.15 uncertain_motif TTTCA in 3 of 20 repeat reads (15%): a pathogenic expansion remains possible; one pathogenic expansion can be sufficient; these reads do not resolve an allele; confirm with repeat-primed PCR if symptoms fit diff --git a/tests/parity/want/rfc1/pathogenic-minority.tsv b/tests/parity/want/rfc1/pathogenic-minority.tsv new file mode 100644 index 00000000..ef0c0fdd --- /dev/null +++ b/tests/parity/want/rfc1/pathogenic-minority.tsv @@ -0,0 +1,2 @@ +locus chrom start end repeat_reads dominant_motif dominant_class dominant_share motif_shares verdict note +RFC1 chr4 101 155 20 AAAAG benign 0.85 AAAAG=0.85,AAGGG=0.15 uncertain_motif AAGGG in 3 of 20 repeat reads (15%): a pathogenic expansion remains possible; CANVAS needs a pathogenic expansion on both copies; these reads do not resolve either copy; confirm with repeat-primed PCR if symptoms fit diff --git a/tests/parity/want/rfc1/unknown-tie.tsv b/tests/parity/want/rfc1/unknown-tie.tsv new file mode 100644 index 00000000..a205a079 --- /dev/null +++ b/tests/parity/want/rfc1/unknown-tie.tsv @@ -0,0 +1,2 @@ +locus chrom start end repeat_reads dominant_motif dominant_class dominant_share motif_shares verdict note +RFC1 chr4 101 155 10 AAAAG benign 0.50 AAAAG=0.50,AGAGG=0.50 uncertain_motif AGAGG in 5 of 10 repeat reads (50%): a pathogenic expansion remains possible; CANVAS needs a pathogenic expansion on both copies; these reads do not resolve either copy; confirm with repeat-primed PCR if symptoms fit From a6f0861fe631545a30b896d6ddf7853d5f95d583 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:12:50 +0100 Subject: [PATCH 3/4] fix(docs): link motif regressions to their source on GitHub The docs site cannot follow a relative link outside its built tree. The strict site check now passes on all 97 pages. Bead: sv-28 --- docs/09b-stranger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/09b-stranger.md b/docs/09b-stranger.md index 89ba9428..918bb83b 100644 --- a/docs/09b-stranger.md +++ b/docs/09b-stranger.md @@ -91,7 +91,7 @@ About a second, measured in three 30-40x WGS runs ([runtime](hardware-requiremen In the Nextflow pipeline, `STRANGER_RFC1` runs the same count after `STRANGER`, in the same image, on the BAM of the samplesheet row (or the one the pipeline aligned), and writes `/expansion_hunter/_rfc1_motif.tsv` only when Stranger flags RFC1. `HTML_REPORT` gets the table, so the Nextflow report shows the same verdict as step 24's. - The count reads the motif; it does not size a long expansion, and a heterozygous AAGGG allele is a carrier state, not CANVAS. CI checks it on made-up reads ([minority regressions](../rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs), `tests/test_rfc1_motif.py`, the parity cases of `rust/crates/og-bam/tests/parity.rs`, `tests/e2e/expansions-and-rfc1-1-motif.sh`); how it does on real carriers, whose reads the aligner may clip or misplace differently, has not been measured. Confirm any `pathogenic_motif` or `uncertain_motif` with motif-aware, flanking-PCR or repeat-primed-PCR testing, and only if cerebellar-ataxia/neuropathy/vestibular symptoms are present. + The count reads the motif; it does not size a long expansion, and a heterozygous AAGGG allele is a carrier state, not CANVAS. CI checks it on made-up reads ([minority regressions](https://github.com/GeiserX/OwnGenome/blob/main/rust/crates/og-bam/tests/i411_rfc1_motif_minority.rs), `tests/test_rfc1_motif.py`, the parity cases of `rust/crates/og-bam/tests/parity.rs`, `tests/e2e/expansions-and-rfc1-1-motif.sh`); how it does on real carriers, whose reads the aligner may clip or misplace differently, has not been measured. Confirm any `pathogenic_motif` or `uncertain_motif` with motif-aware, flanking-PCR or repeat-primed-PCR testing, and only if cerebellar-ataxia/neuropathy/vestibular symptoms are present. With the STRchive catalog, the count takes RFC1's motifs and its two repeats (AAAAG and AAGGG, chr4:39348424-39348483) from the catalog: benign AAAAG and AAAGG, pathogenic AAGGG, ACAGG and CAGGG. STRchive lists AAAGG as pathogenic, and its own notes say why: expansions of 600 or more AAAGG units are seen in CANVAS, while 100 to 380 units are seen in unaffected people ([PMID 37450567](https://pubmed.ncbi.nlm.nih.gov/37450567/)). Short reads cannot size an allele that long, so the count keeps AAAGG benign, as [Cortese et al. 2019](https://pubmed.ncbi.nlm.nih.gov/30926972/) classed it. On the CI's made-up reads the verdicts are the same as with the built-in motifs. - **The other loci where the motif decides.** At the FAME loci (familial adult myoclonic epilepsy: SAMD12, STARD7, MARCHF6, YEATS2, TNRC6A, RAPGEF2, RAI1; [Ishiura et al. 2018](https://pubmed.ncbi.nlm.nih.gov/29507423/)), SCA31 (BEAN1), SCA37 (DAB1) and aFTLD-U (GOLGA8A), disease comes from an inserted motif (TTTCA at the FAME loci), while an expansion of the reference motif is benign. The STRchive catalog names both classes of motif for each. When Stranger flags one of them, step 9b runs the same count (`og str motif --flagged`) with that locus's motifs and writes one row per flagged locus to `_str_motifs.tsv`, with the verdicts of the RFC1 table. A recessive locus's note asks for a pathogenic expansion on both copies. At a dominant locus, one pathogenic allele can be sufficient. A pathogenic minority below the 3-read or 20% floor gives `uncertain_motif`, with its counts in the note, even when the reference motif dominates. A rerun redoes a `not_checked` table, as for RFC1. `STRANGER_RFC1` in Nextflow writes the same file. Both reports show the verdict of each flagged locus in this table, under Repeat Expansions. CI checks it on made-up SAMD12 reads (`tests/e2e/str-catalog-1-strchive.sh`) and on a made-up catalog locus (the parity cases of `rust/crates/og-bam/tests/parity.rs`); how it does on real carriers has not been measured. From dabcdba50a225c60c2f6c2a4a766b474ac8f2dbf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:21:41 +0100 Subject: [PATCH 4/4] ci(e2e): stop shadowing og str motif now that it differs from the Python on purpose The sv-28 fix makes og str motif write uncertain_motif where the old Python wrote benign_motif, and shadow mode compares bytes with no goldens. A minority-motif input at a flagged locus would fail the shadow call. docs/testing.md no longer lists og str motif among the shadowed tools. --- .github/workflows/e2e.yml | 4 +++- docs/testing.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 11fade08..1b308944 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -110,7 +110,9 @@ env: # tools in a case also runs the Python it replaced, from the oracles the # legs materialise, and fails the case when the two disagree. A lane that # ports a tool adds it here for its pull request. - OG_SHADOW: setup-mask-fasta dev-schema dev-contract pgx-cyp2d6-bed pgx-cyp2d6-measure pgx-cyp2d6-check str-motif str-catalog cdna-check comphet-phase + # str-motif left on purpose: og keeps pathogenic and unknown minority + # motif reads as uncertain_motif, which the Python did not (claim sv-28). + OG_SHADOW: setup-mask-fasta dev-schema dev-contract pgx-cyp2d6-bed pgx-cyp2d6-measure pgx-cyp2d6-check str-catalog cdna-check comphet-phase # The shard legs besides the base leg. A full run uses all of them; a run # of picked cases runs fewer when they fit (scripts/ci/e2e-shards.sh count). The cases took 230 # minutes in one job on 2026-11-02 (run 41390434697). diff --git a/docs/testing.md b/docs/testing.md index 7302f594..b4d6604a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -246,7 +246,7 @@ The cold build downloads the crates once; pulling `OG_BUILDER` the first time ad Each port to `og` is checked against the Python it replaces before that Python is deleted. [`scripts/ci/oracle-diff.sh`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/oracle-diff.sh) runs both on the cases of [`tests/parity/cases.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/tests/parity/cases.tsv). It compares their exit codes, stdout, stderr, the outputs the case names and every new file either side wrote, by meaning, through `og dev parity-compare`. A staged input that a tool changes in place is compared only when the case names it as an output. The Python comes back from the commit `tests/parity/oracle.ref` names and runs in its production image. Where `og` fixes the Python on purpose, a golden file under `tests/parity/golden/` says what `og` must write instead. [`tests/parity/README.md`](https://github.com/GeiserX/OwnGenome/blob/main/tests/parity/README.md) says how to add a port. - **Guard, job `parity`** (every pull request, required through `ci-ok`) builds `og` and runs the self-test first. The self-test plants a changed byte, a missing file, an extra file, a wrong exit code, another stdout, a stale `known.tsv` row, a still-valid row and a stale golden, and each must come out as it should. It runs `og dev parity-compare --self-test`, which plants a difference of each compare kind (a changed byte, a 1-ulp float, a reordered key, an int against a float, a missing row and more). It runs the shadow mode of `bin/og` against a fake oracle too. Then the job runs every case and `tests/test_fixitfelix.sh` with `og`. -- **e2e, shadow mode.** The job `og` builds the binaries once, as `scripts/setup.sh --og` does. Each leg downloads them and writes the oracles into `bin/.og/oracle/`. `OG_SHADOW` in `e2e.yml` names the ported tools. Each `og` call of one of them also runs the Python beside it, on the same files. The call fails when the exit codes, stdout or the new top-level files differ, byte for byte. It also fails when either side changes a file that was there before or writes into a directory that was there before, because the shadow cannot compare those writes. stderr is not compared. A call that agrees prints `og shadow (): og and the Python it replaces agree` on stderr. Today that is `og setup mask-fasta` in `fixitfelix-chr21-1`, on chr21 of GRCh38; `og dev schema` in every case that checks a `summary.json`; `og dev contract` in `output-contract-1-hg002`, on the HG002 results of both legs; and the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) wherever a case runs steps 09b, 21 or 32 in bash, on the fixture BAM and on made-up RFC1 reads. The Nextflow modules of those steps pass no `OG_SHADOW` into their containers; the cases that compare a module's table with the bash step's, byte for byte, carry the shadow's proof to them. +- **e2e, shadow mode.** The job `og` builds the binaries once, as `scripts/setup.sh --og` does. Each leg downloads them and writes the oracles into `bin/.og/oracle/`. `OG_SHADOW` in `e2e.yml` names the ported tools. Each `og` call of one of them also runs the Python beside it, on the same files. The call fails when the exit codes, stdout or the new top-level files differ, byte for byte. It also fails when either side changes a file that was there before or writes into a directory that was there before, because the shadow cannot compare those writes. stderr is not compared. A call that agrees prints `og shadow (): og and the Python it replaces agree` on stderr. Today that is `og setup mask-fasta` in `fixitfelix-chr21-1`, on chr21 of GRCh38; `og dev schema` in every case that checks a `summary.json`; `og dev contract` in `output-contract-1-hg002`, on the HG002 results of both legs; and the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) wherever a case runs steps 21 or 32 in bash, on the fixture BAM. The Nextflow modules of those steps pass no `OG_SHADOW` into their containers; the cases that compare a module's table with the bash step's, byte for byte, carry the shadow's proof to them. The RFC1 motif count (`og str motif`) is not shadowed: it differs from the Python on purpose. When a benign motif carries most reads and any pathogenic or unknown motif reads remain, `og` writes `uncertain_motif`, where the Python wrote `benign_motif`. Its integration tests in `rust/crates/og-bam/tests/` check it against the expected tables under `tests/parity/want/rfc1/`. The first port is the reference mask of `setup.sh --masked-reference`. On a 254 MB synthetic FASTA with 300 rows (the case `setup-mask-fasta/synthetic-254mb`), on an Apple Silicon Mac mini: