From 977e3e28c1f7cf4583f36b7f04d96b5105e66340 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:10:42 +0100 Subject: [PATCH 1/5] test(longread): step 04c must refuse short reads and pass a tandem-repeat map The fake-docker cases fail on main: step 04c runs Sniffles2 on a short-read BAM and exits 0, never passes --tandem-repeats, and its closing hint sends long-read SVs to duphold for a GQ. Bead: sv-18 Bead: sv-17 --- .../fake-docker/bash-step-partial-outputs.sh | 8 +++- tests/fake-docker/bash-step-sv.sh | 46 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/tests/fake-docker/bash-step-partial-outputs.sh b/tests/fake-docker/bash-step-partial-outputs.sh index d099ef108..ed65d3fe3 100755 --- a/tests/fake-docker/bash-step-partial-outputs.sh +++ b/tests/fake-docker/bash-step-partial-outputs.sh @@ -6,7 +6,9 @@ # exists". An empty VCF left by an older version is redone too. # Sniffles2 (04c): the tool writes part of its VCF and exits 1; the rerun # must replace it (Sniffles2 refuses an existing output without -# --allow-overwrite). +# --allow-overwrite). Its closing hints do not send long-read SVs to +# duphold for a GQ: Sniffles2 writes its own, and step 15 reads Manta's +# calls only. # Clair3 (03e): the tool exits 0 without merge_output.vcf.gz while an older # sample1.vcf.gz is present. The step fails instead of printing "complete". # FreeBayes (03b): bcftools sort writes part of its VCF and fails. The step @@ -51,6 +53,9 @@ word_after() { return 1 } case "$args" in + *"samtools view -F 0x900 "*) + # Step 04c's read-length check: 10,000 reads of 15 kb, a long-read BAM. + printf '10000 15000\n' ;; *" stranger "*) if [ "${FAKE_MODE:-ok}" = fail ]; then printf '##fileformat=VCFv4.2\n'; exit 1; fi printf "${header}${record}" ;; @@ -114,6 +119,7 @@ SV="${GENOME_DIR}/sample1/sv_sniffles/sample1_sv.vcf.gz" FAKE_MODE=fail run_rc sniffles-fail "${SCRIPTS}/04c-sniffles2.sh" sample1 [ "$RC" -ne 0 ] || fail "step 04c exited 0 although Sniffles2 failed" run_expect 0 sniffles-rerun "${SCRIPTS}/04c-sniffles2.sh" sample1 +output_lacks sniffles-rerun 'GQ scores' have_bgzf_eof() { [ "$(tail -c 28 "$1" | od -An -v -tx1 | tr -d ' \n')" = 1f8b08040000000000ff0600424302001b0003000000000000000000 ]; } have_bgzf_eof "$SV" || fail "the rerun of step 04c did not replace the half-written ${SV}" diff --git a/tests/fake-docker/bash-step-sv.sh b/tests/fake-docker/bash-step-sv.sh index 13173fd89..35925efdd 100755 --- a/tests/fake-docker/bash-step-sv.sh +++ b/tests/fake-docker/bash-step-sv.sh @@ -10,6 +10,11 @@ # than GRIDSS_MIN_MEM_GB (default 32; the fake Docker reports 16 GiB). # Delly (19): gets -x with the exclude map setup.sh installs; without it, # runs as before and warns. +# Sniffles2 (04c): stops with exit 1, before Sniffles2, when the first +# primary reads of the BAM average under 1,000 bp (Sniffles2 finds no SV +# in short reads and exits 0 with an empty VCF), or when it has none; +# gets --tandem-repeats with the tandem-repeat map setup.sh installs; +# without it, runs as before and says so. # duphold (15): writes duphold/_sv_filtered.vcf.gz with the filter # expression of the Nextflow DUPHOLD_FILTER, and its index. # AnnotSV (05): builds its input from that file, the input of the Nextflow @@ -31,6 +36,15 @@ set -euo pipefail . "${CASE_WORK:?}/host-path.sh" args=" ${*:2} " case "$args" in + *"samtools view -F 0x900 "*) + # Step 04c's read-length check: reads read, and their mean length. + printf '%s\n' "${FAKE_READS:-10000 15000}" ;; + *" sniffles "*) + v=$(sed -n 's/.* -v \([^ ]*\) .*/\1/p' <<<"$args") + h=$(host_path "$v") + mkdir -p "$(dirname "$h")" + printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n' | gzip -c > "$h" + : > "${h}.tbi" ;; *" tiddit "*) if [ "${FAKE_TIDDIT_NO_VCF:-}" = 1 ]; then echo "fake TIDDIT: the BAM index does not match the BAM; stopping" @@ -93,6 +107,38 @@ printf 'chr1\t0\t10000\ttelomere\n' > "${GENOME_DIR}/reference/delly_human.hg38. run_expect 0 delly "${SCRIPTS}/19-delly.sh" sample1 docker_log_has '^run image=[^ ]*delly.* delly sr .*-x /genome/reference/delly_human\.hg38\.excl\.tsv ' "Delly ran without its exclude map" +# --- Sniffles2 ------------------------------------------------------------------------ +mkdir -p "${GENOME_DIR}/sample1/aligned_longread" +cp "${GENOME_DIR}/sample1/aligned/"* "${GENOME_DIR}/sample1/aligned_longread/" +sniffles_ran() { awk '/^run image=[^ ]*sniffles/ && / sniffles / { found = 1 } END { exit !found }' "$FAKE_DOCKER_LOG"; } +# A short-read BAM: the step stops before Sniffles2 and names the fix. +: > "$FAKE_DOCKER_LOG" +FAKE_READS='10000 151' run_rc sniffles-short "${SCRIPTS}/04c-sniffles2.sh" sample1 +expect_rc sniffles-short 1 +output_has sniffles-short 'average 151 bp' +output_has sniffles-short 'Point ALIGN_DIR at the long-read BAM' +docker_log_has '^run image=[^ ]*samtools[^ ]* .*samtools.*view.*-F.*0x900' "step 04c did not read the read lengths with samtools" +if sniffles_ran; then fail "step 04c ran Sniffles2 on a short-read BAM"; fi +# No primary read with a sequence: an empty BAM, or one samtools cannot read. +: > "$FAKE_DOCKER_LOG" +FAKE_READS='0 0' run_rc sniffles-no-reads "${SCRIPTS}/04c-sniffles2.sh" sample1 +expect_rc sniffles-no-reads 1 +output_has sniffles-no-reads 'no primary read' +if sniffles_ran; then fail "step 04c ran Sniffles2 on a BAM with no read"; fi +# Long reads without the tandem-repeat map: Sniffles2 runs without it, with a note. +: > "$FAKE_DOCKER_LOG" +run_expect 0 sniffles-no-tr "${SCRIPTS}/04c-sniffles2.sh" sample1 +sniffles_ran || fail "step 04c did not run Sniffles2 on a long-read BAM" +output_has sniffles-no-tr 'NOTE: no tandem-repeat map' +if grep -q -- '--tandem-repeats' "$FAKE_DOCKER_LOG"; then fail "Sniffles2 got --tandem-repeats although no map is installed"; fi +# With the map setup.sh installs (the name of TANDEM_REPEATS_URL, unpacked). +printf 'chr1\t10000\t11014\n' > "${GENOME_DIR}/reference/GRCh38_AllTandemRepeats.bed" +: > "$FAKE_DOCKER_LOG" +run_expect 0 sniffles-tr "${SCRIPTS}/04c-sniffles2.sh" sample1 +docker_log_has '^run image=[^ ]*sniffles[^ ]* .* sniffles .*--tandem-repeats /genome/reference/GRCh38_AllTandemRepeats\.bed( |$)' \ + "Sniffles2 ran without the tandem-repeat map setup.sh installs" +output_lacks sniffles-tr 'NOTE: no tandem-repeat map' + # --- duphold -------------------------------------------------------------------------- mkdir -p "${GENOME_DIR}/sample1/manta/results/variants" printf 'placeholder\n' > "${GENOME_DIR}/sample1/manta/results/variants/diploidSV.vcf.gz" From 52ef85619d768bfb0ba893ffca13be5098d14df6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:14:23 +0100 Subject: [PATCH 2/5] fix(longread): refuse short reads in Sniffles2 and give it a tandem-repeat map Step 04c ran Sniffles2 on any BAM. On short reads Sniffles2 finds no SV and still exits 0, so the step reported success with an empty VCF. It now reads the mean length of the first 10,000 primary reads with the pinned samtools image and stops below 1,000 bp, naming the long-read BAM to use. setup.sh installs GIAB's GRCh38 tandem-repeat stratification (v3.6, NIST data, pinned by sha256) and keeps it unpacked; step 04c passes it as --tandem-repeats when present and says so when not. Sniffles' own file comes from PacBio's pbsv under a licence limited to PacBio data, so it could not serve ONT reads. The closing hint no longer sends long-read SVs to duphold for a GQ: Sniffles2 writes its own, and step 15 reads Manta's calls only. Bead: sv-18 Bead: sv-17 Bead: D14 --- scripts/04c-sniffles2.sh | 50 ++++++++++++++++++++++++++++++++++++---- scripts/setup.sh | 33 ++++++++++++++++++++++++-- versions.env | 9 ++++++++ 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/scripts/04c-sniffles2.sh b/scripts/04c-sniffles2.sh index 3fc81ea44..99528a1e1 100755 --- a/scripts/04c-sniffles2.sh +++ b/scripts/04c-sniffles2.sh @@ -10,8 +10,13 @@ # Sniffles2 writes the bgzipped VCF and its .tbi itself (an -v name ending in # .vcf.gz), and --allow-overwrite lets a rerun replace the output of a run # that failed half way. -# Known gap: Sniffles2 calls SVs in tandem repeats more accurately with a -# tandem-repeat BED (--tandem-repeats); none is passed here yet. +# +# Sniffles2 finds no SV in short reads and still exits 0 with an empty VCF, +# so the step first reads the mean length of the first 10,000 primary reads +# (SAMTOOLS_IMAGE) and stops below 1,000 bp. With the tandem-repeat map +# setup.sh installs (TANDEM_REPEATS_URL, unpacked in reference/), Sniffles2 +# gets --tandem-repeats, which gives SVs inside repeats better breakpoints +# and merging; without it, the step runs as before and says so. set -euo pipefail SAMPLE=${1:?Usage: $0 } @@ -41,6 +46,42 @@ for f in "$BAM" "${BAM}.bai" "$REF" "${REF}.fai"; do fi done +# Long reads only: the mean length of the first 10,000 primary reads with a +# sequence, as " ". +MIN_MEAN_READ_LEN=1000 +# shellcheck disable=SC2016 # the program and $1/$2 belong to the inner shell +LEN_AWK='$10 != "*" { n++; s += length($10); if (n == 10000) exit } END { printf "%d %d\n", n, (n ? s / n : 0) }' +# shellcheck disable=SC2016 +PROBE=$(run_in "$SAMTOOLS_IMAGE" sh -c 'samtools view -F 0x900 "$1" | awk -F "\t" "$2"' sh \ + "/genome/${SAMPLE}/${ALIGN_DIR}/${SAMPLE}_sorted.bam" "$LEN_AWK") +read -r N_READS MEAN_LEN _ <<< "$PROBE" || true +if ! [[ "${N_READS:-}" =~ ^[0-9]+$ && "${MEAN_LEN:-}" =~ ^[0-9]+$ ]]; then + echo "ERROR: could not read the read lengths of ${BAM} (samtools printed '${PROBE}')" >&2 + exit 1 +fi +if [ "$N_READS" -eq 0 ]; then + echo "ERROR: ${BAM} has no primary read with a sequence (see samtools's message above, if any); Sniffles2 has nothing to call SVs from." >&2 + exit 1 +fi +if [ "$MEAN_LEN" -lt "$MIN_MEAN_READ_LEN" ]; then + echo "ERROR: the first ${N_READS} primary reads of ${BAM} average ${MEAN_LEN} bp." >&2 + echo " Sniffles2 needs long reads (ONT or PacBio HiFi, ${MIN_MEAN_READ_LEN} bp or more on average): on short reads it finds no SV and still exits 0." >&2 + echo " Point ALIGN_DIR at the long-read BAM (step 02b writes aligned_longread/${SAMPLE}_sorted.bam). Short-read SVs come from steps 04 (Manta), 04a, 04b and 19." >&2 + exit 1 +fi +echo "Reads: the first ${N_READS} primary reads average ${MEAN_LEN} bp" + +# The tandem-repeat map setup.sh installs: TANDEM_REPEATS_URL's file, unpacked. +TR_NAME=${TANDEM_REPEATS_URL##*/} +TR_BED="${GENOME_DIR}/reference/${TR_NAME%.gz}" +TR_ARGS=() +if [ -f "$TR_BED" ] && [ -s "$TR_BED" ]; then + TR_ARGS=(--tandem-repeats "$(cpath "$TR_BED")") + echo "Tandem repeats: ${TR_BED}" +else + echo "NOTE: no tandem-repeat map at ${TR_BED} (setup.sh installs it): Sniffles2 runs without --tandem-repeats, so SVs inside repeats get less precise breakpoints and merging." +fi + mkdir -p "$OUTPUT_DIR" # Run Sniffles2 @@ -54,7 +95,8 @@ run_in --cpus "$THREADS" --memory 16g \ --reference "${REF_FASTA_C}" \ --threads "$THREADS" \ --sample-id "${SAMPLE}" \ - --allow-overwrite + --allow-overwrite \ + ${TR_ARGS[@]+"${TR_ARGS[@]}"} if ! wrote_vcf "${OUTPUT_DIR}/${SAMPLE}_sv.vcf.gz"; then echo "ERROR: Sniffles2 exited without a ${OUTPUT_DIR}/${SAMPLE}_sv.vcf.gz" >&2 @@ -88,4 +130,4 @@ echo "" echo "Next steps:" echo " - Annotate SVs: ./scripts/05-annotsv.sh ${SAMPLE} (set SV_VCF to point at sv_sniffles)" echo " - Merge with short-read SVs: ./scripts/22-survivor-merge.sh ${SAMPLE}" -echo " - Filter quality: duphold can re-annotate long-read SVs with GQ scores" +echo " - Quality: Sniffles2 writes its own QUAL and FORMAT/GQ; step 15 (duphold) reads Manta's short-read calls only" diff --git a/scripts/setup.sh b/scripts/setup.sh index 326dfeddd..1dbc04950 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -73,8 +73,9 @@ # GENCODE gene coordinates T1K needs (~350 MB) and GENCODE's exons for # the cDNA read check of steps 06 and 17, both from one ~80 MB download, # and somalier's sites and VerifyBamID2's marker panel for step 33 (~10 MB), -# and gnomAD v3.1's mitochondrial sites for step 20 (~5 MB), and the -# STRchive repeat catalog of steps 09 and 9b (~0.5 MB) +# and gnomAD v3.1's mitochondrial sites for step 20 (~5 MB), the +# STRchive repeat catalog of steps 09 and 9b (~0.5 MB), and GIAB's +# tandem-repeat map for Sniffles2 in step 04c (~9 MB, ~35 MB unpacked) # 5. AnnotSV annotation data for step 5 (~5.3 GB download, ~20 GB unpacked), # plus gnomAD-SV's common SVs (~1.7 GB download, kept as a few MB) and the # MANE Select transcript list @@ -717,6 +718,33 @@ install_strchive_data() { echo "[OK] STRchive ${STRCHIVE_VERSION} catalog (steps 09 and 9b): ${dir}/catalog.json" } +# install_tandem_repeats: the tandem-repeat map step 04c passes to Sniffles2 +# (--tandem-repeats): TANDEM_REPEATS_URL, checked against +# TANDEM_REPEATS_SHA256, and its unpacked copy beside it, because Sniffles2 +# reads a plain BED only. The copy is written under a temporary name first, +# so a file under the final name is whole. Without the map step 04c runs as +# before and says so. +install_tandem_repeats() { + local gz="${GENOME_DIR}/reference/${TANDEM_REPEATS_URL##*/}" + local bed="${gz%.gz}" + if [ -s "$bed" ] && [ -s "$gz" ] && [ "$(_digest sha256 "$gz")" = "$TANDEM_REPEATS_SHA256" ]; then + echo "[OK] GIAB tandem-repeat map (step 04c) already present." + return 0 + fi + rm -f "$bed" + if ! fetch "$TANDEM_REPEATS_URL" "$gz" sha256 "$TANDEM_REPEATS_SHA256"; then + echo "[WARN] Could not install GIAB's tandem-repeat map (${TANDEM_REPEATS_URL}). Step 04c runs Sniffles2 without it; re-run setup.sh to try again." + return 1 + fi + if ! gzip -dc "$gz" > "${bed}.part" || [ ! -s "${bed}.part" ]; then + rm -f "${bed}.part" + echo "[WARN] Could not unpack ${gz}. Step 04c runs Sniffles2 without the tandem-repeat map; re-run setup.sh to try again." + return 1 + fi + mv -f "${bed}.part" "$bed" + echo "[OK] GIAB tandem-repeat map (step 04c): ${bed}" +} + # install_consumer_sites: the site table of the raw-data export (step 40). The # 23andMe v5 site list (GRCh37 positions and reference bases) of # CONSUMER_SITES_COMMIT, checked against CONSUMER_SITES_SHA256, is joined by @@ -2445,6 +2473,7 @@ fi # A failed download does not stop setup here either. install_sample_qc_data || true install_strchive_data || true +install_tandem_repeats || true # Step 25: its scores, pgsc_calc and its plugin (warnings only). install_prs diff --git a/versions.env b/versions.env index 304024130..331331c51 100644 --- a/versions.env +++ b/versions.env @@ -219,6 +219,15 @@ MANE_SUMMARY_SHA256="d10ace2720681a3b2e0eefd9da4f551274a6b4141ac9bfd6a2565dfb6e9 # file as downloaded. Change the two together. GNOMAD_MITO_URL="https://storage.googleapis.com/gcp-public-data--gnomad/release/3.1/vcf/genomes/gnomad.genomes.v3.1.sites.chrM.vcf.bgz" GNOMAD_MITO_SHA256="507e4a1fab8c8d2dfb5d77239b0fb31e3db92634519f1d6cd9b9ff38df181f99" +# The tandem-repeat map of step 04c (Sniffles2 --tandem-repeats): every +# GRCh38 tandem repeat of GIAB's genome stratifications v3.6 (TRF and +# RepeatMasker simple repeats, 5 bp added on each side), NIST data free to +# reuse. Sniffles' own GRCh38 file comes from PacBio's pbsv under a licence +# that allows PacBio data only, so ONT reads could not use it. setup.sh checks +# the download against the sha256 and keeps it unpacked beside it in +# reference/: Sniffles2 reads a plain BED only. Change the two together. +TANDEM_REPEATS_URL="https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/genome-stratifications/v3.6/GRCh38@all/LowComplexity/GRCh38_AllTandemRepeats.bed.gz" +TANDEM_REPEATS_SHA256="ff58710eb3aab81c172f83ac263ab698068b0a4f0d8357d7347ff9a7b8774179" # STRchive (github.com/dashnowlab/STRchive, CC BY 4.0), the disease repeat # catalog steps 09 and 9b run on: setup.sh downloads its ExpansionHunter # catalog and its loci file from the release, checks each against its sha256 From 7cc487c602c9433db04cbc6989b3a8737851dff0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:30:33 +0100 Subject: [PATCH 3/5] test(smoke): check the SV type, size and filter, and Delly's sex model on chrX The SV rows passed on any record within 500 bp of the planted deletion, of any type, size or filter, and nothing caught false calls elsewhere. Sniffles2, Manta, Delly and TIDDIT now need one PASS deletion of 2.7 to 3.3 kb there; Sniffles2 also needs no PASS record elsewhere, and a second row runs it with --tandem-repeats as step 04c does. GRIDSS keeps the loose check: it writes breakends, not DEL records. A new DELLY_IMAGE row runs delly sr --sex male and female on mini.bam moved onto a chrX of GRCh38's length past PAR1 (input chrx): the male genotype must be haploid, the female 1/1, and an unknown value must fail. Delly ignores --sex without a word on a chrX of another length; the row goes red when the chrX is one base longer. The self-test plants records for each rule of the new checks. Bead: sv-43 Bead: sv-41 --- scripts/ci/image-smoke.sh | 103 +++++++++++++++++++++++++++++++++++++- tests/smoke/commands.tsv | 11 ++-- tests/smoke/delly_sex.sh | 17 +++++++ 3 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 tests/smoke/delly_sex.sh diff --git a/scripts/ci/image-smoke.sh b/scripts/ci/image-smoke.sh index 3457a5398..cb9360c1f 100755 --- a/scripts/ci/image-smoke.sh +++ b/scripts/ci/image-smoke.sh @@ -49,7 +49,7 @@ ROW_TIMEOUT=${ROW_TIMEOUT:-3600} GH_REPO=${GITHUB_REPOSITORY:-GeiserX/OwnGenome} TAG=$(tr -d '[:space:]' < "${REPO}/tests/fixtures/VERSION") SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} -NEEDS_KNOWN="ref reads bam longreads longbam truth fullref slice vcf vcf50 pgxvcf revel sv dels bundle cyrius mito hlareads qc ehcatalog chain annotsv" +NEEDS_KNOWN="ref reads bam longreads longbam truth fullref slice vcf vcf50 pgxvcf revel sv dels bundle cyrius mito hlareads qc ehcatalog chain annotsv chrx" OPTS_KNOWN="root net full also= rw=" # The mini reference: chr20:10,000,001-10,500,000 of the fixture reference as @@ -61,6 +61,10 @@ MINI_LEN=503000 PLANT_POS=450000 PLANT_LEN=3000 SMALL_END=449000 +# chrX for Delly's sex model (need_chrx): GRCh38's chrX length, and where the +# mini reference starts on it, past the end of PAR1 (2,781,479). +X_LEN=156040895 +X_SHIFT=3000000 # shellcheck source=../../versions.env . "$VERSIONS" @@ -146,6 +150,23 @@ self_test() { if grep -q '^\[PASS\] true filter' <<< "$out" && grep -q '^\[FAIL\] false filter' <<< "$out" && grep -q '^fails=1$' <<< "$out"; then echo "[PASS] jqcheck rejects a false filter" else echo "[FAIL] jqcheck rejects a false filter: $out"; fails=$((fails + 1)); fi + # The SV checks on planted records: only a PASS deletion of the right size + # at the planted deletion counts, and only a PASS record elsewhere is false. + local del pass="PASS" + rec() { printf 'chr20\t%s\t.\tN\t%s\t.\t%s\t%s\n' "$@"; } + sv_case() { # sv_case DESCRIPTION WANT VALUE + if [ "$3" = "$2" ]; then echo "[PASS] $1"; else echo "[FAIL] $1 (got '$3', want '$2')"; fails=$((fails + 1)); fi + } + del="SVTYPE=DEL;SVLEN=-3000;END=$((PLANT_POS + PLANT_LEN))" + sv_case "a PASS 3 kb deletion at the planted one counts" 1 "$(rec "$PLANT_POS" '' "$pass" "$del" | sv_del_rows 2700 3300 "$PLANT_POS")" + sv_case "a deletion with END and no SVLEN counts" 1 "$(rec "$((PLANT_POS + 2))" '' . "SVTYPE=DEL;END=$((PLANT_POS + PLANT_LEN))" | sv_del_rows 2700 3300 "$PLANT_POS")" + sv_case "a 3 kb insertion there does not count" 0 "$(rec "$((PLANT_POS + 300))" '' "$pass" 'SVTYPE=INS;SVLEN=3000' | sv_del_rows 2700 3300 "$PLANT_POS")" + sv_case "a 60 bp deletion there does not count" 0 "$(rec "$PLANT_POS" '' "$pass" 'SVTYPE=DEL;SVLEN=-60' | sv_del_rows 2700 3300 "$PLANT_POS")" + sv_case "a filtered deletion there does not count" 0 "$(rec "$PLANT_POS" '' LowQual "$del" | sv_del_rows 2700 3300 "$PLANT_POS")" + sv_case "a deletion 2 kb away does not count" 0 "$(rec "$((PLANT_POS - 2000))" '' "$pass" "$del" | sv_del_rows 2700 3300 "$PLANT_POS")" + sv_case "a PASS record 2 kb away is a false call" 1 "$(rec "$((PLANT_POS - 2000))" '' "$pass" 'SVTYPE=INS;SVLEN=60' | sv_far_rows "$PLANT_POS")" + sv_case "a filtered record away is not" 0 "$(rec 100000 '' GT 'SVTYPE=INS;SVLEN=60' | sv_far_rows "$PLANT_POS")" + sv_case "the planted deletion is not" 0 "$(rec "$PLANT_POS" '' "$pass" "$del" | sv_far_rows "$PLANT_POS")" # Read the production filters, so a weakened assertion makes this test red. local counts catalog genotype copies depth counts=$(awk -F '\t' '$1 == "FASTP_IMAGE" {print $5}' "$TABLE" | cut -d "'" -f 2) @@ -534,6 +555,46 @@ need_chain() { printf 'chain 1000 chr20 %s + 0 %s chr20 %s + 0 %s 1\n%s\n\n' \ "$MINI_LEN" "$SMALL_END" "$MINI_LEN" "$SMALL_END" "$SMALL_END" > "${IN}/identity.chain" } +# chrX for Delly's sex model. Delly applies --sex only on a chrX of a known +# build length (156,040,895 bp for GRCh38) and treats PAR1 as diploid, so +# xmini.fa is that chrX: N up to X_SHIFT, then mini.fa (padded with N to a +# full line), then N to the end. xmini.bam is mini.bam moved there: contig, +# positions, mate positions and SA tags shifted by X_SHIFT. +need_chrx() { + need ref && need bam || return 1 + awk -v pre="$X_SHIFT" -v len="$X_LEN" ' + NR > 1 { seq = seq $0 } + function ns(k, i, l) { l = ""; for (i = 0; i < k; i++) l = l "N"; return l } + END { + w = 60; full = ns(w); print ">chrX" + for (i = 0; i < pre / w; i++) print full + n = length(seq) + for (i = 1; i <= n; i += w) { l = substr(seq, i, w); if (length(l) < w) l = l ns(w - length(l)); print l } + rest = len - pre - int((n + w - 1) / w) * w + for (i = 0; i < int(rest / w); i++) print full + if (rest % w) print ns(rest % w) + }' "${IN}/mini.fa" > "${IN}/xmini.fa" || return 1 + hsam faidx /in/xmini.fa || return 1 + [ "$(cut -f2 "${IN}/xmini.fa.fai")" = "$X_LEN" ] || return 1 + # shellcheck disable=SC2016 # the program belongs to the container + hrun "$SAMTOOLS_IMAGE" sh -c 'samtools view -h /in/mini.bam | awk -v s="$1" -v len="$2" "$3" | samtools view -b -o /in/xmini.bam - && samtools index /in/xmini.bam' \ + sh "$X_SHIFT" "$X_LEN" ' + BEGIN { FS = OFS = "\t" } + /^@SQ/ { print "@SQ", "SN:chrX", "LN:" len; next } + /^@/ { print; next } + { + if ($3 == "chr20") { $3 = "chrX"; $4 += s } + if ($7 == "chr20") { $7 = "chrX"; $8 += s } else if ($7 == "=" && $8 > 0) $8 += s + for (i = 12; i <= NF; i++) if ($i ~ /^SA:Z:/) { + m = split(substr($i, 6), a, ";"); t = "SA:Z:" + for (j = 1; j <= m; j++) if (a[j] != "") { split(a[j], f, ","); if (f[1] == "chr20") { f[1] = "chrX"; f[2] += s } + t = t f[1] "," f[2] "," f[3] "," f[4] "," f[5] "," f[6] ";" } + $i = t + } + print + }' || return 1 + [ "$(hsam view -c /in/xmini.bam)" = "$(hsam view -c /in/mini.bam)" ] +} # AnnotSV's annotation data, as setup.sh fetches it (5.3 GB; the server is slow). need_annotsv() { local url="https://www.lbgi.fr/~geoffroy/Annotations/Annotations_Human_${ANNOTSV_ANNOTATIONS_VERSION}.tar.gz" @@ -617,6 +678,46 @@ sv_near() { bcf view -H "$1" | awk -v a="$PLANT_POS" -v b=$((PLANT_POS + PLANT_LEN)) \ '{d1 = $2 - a; d2 = $2 - b; if ((d1 < 0 ? -d1 : d1) <= 500 || (d2 < 0 ? -d2 : d2) <= 500) n++} END {print n + 0}' } +# sv_del_near VCF|BCF MIN MAX: deletions at the planted deletion (within +# 500 bp of either end) with FILTER PASS or none and a length from MIN to MAX +# bp (|SVLEN|, or END - POS without one). sv_near counts any record there. +sv_del_near() { bcf view -H "$1" | sv_del_rows "$2" "$3" "$PLANT_POS"; } +# sv_far_pass VCF|BCF: PASS records farther than 500 bp from both ends of the +# planted deletion. The reads carry no other SV, so each one is a false call. +sv_far_pass() { bcf view -H "$1" | sv_far_rows "$PLANT_POS"; } +# sv_gt_near VCF|BCF START: the genotype of the first deletion within 500 bp +# of START or START + PLANT_LEN. +sv_gt_near() { + bcf query -f '%POS\t%INFO/SVTYPE\t[%GT]\n' "$1" | awk -F'\t' -v a="$2" -v b=$(($2 + PLANT_LEN)) ' + function abs(x) { return x < 0 ? -x : x } + $2 == "DEL" && (abs($1 - a) <= 500 || abs($1 - b) <= 500) { print $3; exit }' +} +# sv_del_rows MIN MAX START and sv_far_rows START read VCF records on stdin; +# START is where the planted deletion begins on the record's contig. +sv_del_rows() { + awk -F'\t' -v lo="$1" -v hi="$2" -v a="$3" -v b=$(($3 + PLANT_LEN)) ' + function abs(x) { return x < 0 ? -x : x } + $7 == "PASS" || $7 == "." { + t = ""; len = ""; end = "" + k = split($8, kv, ";") + for (i = 1; i <= k; i++) { + if (kv[i] ~ /^SVTYPE=/) t = substr(kv[i], 8) + else if (kv[i] ~ /^SVLEN=/) len = substr(kv[i], 7) + else if (kv[i] ~ /^END=/) end = substr(kv[i], 5) + } + if (t != "DEL") next + if (len == "") len = (end == "" ? 0 : end - $2) + len = abs(len + 0) + if ((abs($2 - a) <= 500 || abs($2 - b) <= 500) && len >= lo && len <= hi) n++ + } + END { print n + 0 }' +} +sv_far_rows() { + awk -F'\t' -v a="$1" -v b=$(($1 + PLANT_LEN)) ' + function abs(x) { return x < 0 ? -x : x } + $7 == "PASS" && abs($2 - a) > 500 && abs($2 - b) > 500 { n++ } + END { print n + 0 }' +} # vcf_field VCF ID KEY: the INFO value, or else the first sample's FORMAT value, # of KEY in the record whose ID column is ID. vcf_field() { diff --git a/tests/smoke/commands.tsv b/tests/smoke/commands.tsv index 039419d5d..d6511768c 100644 --- a/tests/smoke/commands.tsv +++ b/tests/smoke/commands.tsv @@ -14,12 +14,15 @@ FREEBAYES_IMAGE - ref,bam,truth freebayes -f /in/mini.fa --region chr20:1-449000 STRELKA_IMAGE - ref,bam,truth configureStrelkaGermlineWorkflow.py --bam /in/mini.bam --referenceFasta /in/mini.fa --runDir strelka && strelka/runWorkflow.py -m local -j 4 at_least "SNV recall against the GIAB truth" "$(snv_recall strelka/results/variants/variants.vcf.gz)" 0.9 OCTOPUS_IMAGE - ref,bam,truth octopus -R /in/mini.fa -I /in/mini.bam -o octopus.vcf.gz --threads 4 --regions chr20:1-449000 at_least "SNV recall against the GIAB truth" "$(snv_recall octopus.vcf.gz)" 0.9 CLAIR3_IMAGE - ref,longbam,truth /opt/bin/run_clair3.sh --bam_fn=/in/long.bam --ref_fn=/in/mini.fa --platform=hifi --model_path=/opt/models/hifi_revio --output=/out/clair3 --threads=4 --sample_name=HG002 --ctg_name=chr20 && { ls -d /opt/models/r1041_e82_400bps_sup_v500 /opt/models/hifi_revio > models.txt 2>&1 || true; } at_least "SNV recall against the GIAB truth (synthetic HiFi reads)" "$(snv_recall clair3/merge_output.vcf.gz)" 0.8; contains "the ONT model directory step 03e uses" models.txt "^/opt/models/r1041_e82_400bps_sup_v500$"; contains "the HiFi model directory step 03e uses" models.txt "^/opt/models/hifi_revio$" -TIDDIT_IMAGE - ref,bam tiddit --sv --bam /in/mini.bam --ref /in/mini.fa --threads 4 --skip_assembly -o tiddit at_least "records at the planted deletion" "$(sv_near tiddit.vcf)" 1 +TIDDIT_IMAGE - ref,bam tiddit --sv --bam /in/mini.bam --ref /in/mini.fa --threads 4 --skip_assembly -o tiddit same "PASS deletions of 2,700-3,300 bp at the planted deletion" "$(sv_del_near tiddit.vcf 2700 3300)" 1 # Sniffles skips contigs shorter than 1 Mb unless told otherwise; the mini reference is 503 kb. -SNIFFLES_IMAGE - ref,longbam sniffles -i /in/long.bam -v sniffles.vcf --reference /in/mini.fa --threads 4 --sample-id HG002 --all-contigs at_least "records at the planted deletion" "$(sv_near sniffles.vcf)" 1 +SNIFFLES_IMAGE - ref,longbam sniffles -i /in/long.bam -v sniffles.vcf --reference /in/mini.fa --threads 4 --sample-id HG002 --all-contigs same "PASS deletions of 2,700-3,300 bp at the planted deletion" "$(sv_del_near sniffles.vcf 2700 3300)" 1; same "PASS records away from the planted deletion (the reads carry no other SV)" "$(sv_far_pass sniffles.vcf)" 0 +# Step 04c passes --tandem-repeats with the map setup.sh installs; here a repeat over the planted deletion. +SNIFFLES_IMAGE - ref,longbam printf 'chr20\t449500\t453500\n' > tr.bed && sniffles -i /in/long.bam -v sniffles.vcf --reference /in/mini.fa --threads 4 --sample-id HG002 --all-contigs --tandem-repeats tr.bed > sniffles.log 2>&1; rc=$?; cat sniffles.log; exit $rc contains "Sniffles2 read the tandem-repeat map" sniffles.log "tandem repeat annotations"; same "PASS deletions of 2,700-3,300 bp at the planted deletion" "$(sv_del_near sniffles.vcf 2700 3300)" 1; same "PASS records away from the planted deletion" "$(sv_far_pass sniffles.vcf)" 0 HAPPY_IMAGE root ref,truth /opt/hap.py/bin/hap.py /in/truth.vcf.gz /in/query90.vcf.gz -r /in/mini.fa -f /in/truth.bed -o happy --engine=vcfeval between "SNP recall of a query missing every tenth truth SNV" "$(csv_cell happy.summary.csv Type=SNP,Filter=PASS METRIC.Recall)" 0.85 0.95; at_least "SNP precision of that query" "$(csv_cell happy.summary.csv Type=SNP,Filter=PASS METRIC.Precision)" 0.99 -MANTA_IMAGE - ref,bam configManta.py --bam /in/mini.bam --referenceFasta /in/mini.fa --runDir manta && manta/runWorkflow.py -j 4 at_least "records at the planted deletion" "$(sv_near manta/results/variants/diploidSV.vcf.gz)" 1 -DELLY_IMAGE - ref,bam delly sr -g /in/mini.fa -o delly.bcf /in/mini.bam at_least "records at the planted deletion" "$(sv_near delly.bcf)" 1 +MANTA_IMAGE - ref,bam configManta.py --bam /in/mini.bam --referenceFasta /in/mini.fa --runDir manta && manta/runWorkflow.py -j 4 same "PASS deletions of 2,700-3,300 bp at the planted deletion" "$(sv_del_near manta/results/variants/diploidSV.vcf.gz 2700 3300)" 1 +DELLY_IMAGE - ref,bam delly sr -g /in/mini.fa -o delly.bcf /in/mini.bam same "PASS deletions of 2,700-3,300 bp at the planted deletion" "$(sv_del_near delly.bcf 2700 3300)" 1 +DELLY_IMAGE - chrx sh /smoke/delly_sex.sh same "genotype of the planted deletion on a male chrX (haploid)" "$(sv_gt_near male.bcf $((X_SHIFT + PLANT_POS)))" 1; same "and on a female chrX" "$(sv_gt_near female.bcf $((X_SHIFT + PLANT_POS)))" 1/1; contains "Delly used --sex male" male.log "sex=male"; contains "Delly used --sex female" female.log "sex=female"; same "--sex bogus" "$(cat bogus.txt)" rejected CNVPYTOR_IMAGE root ref,bam sh /smoke/cnvpytor.sh at_least "CNVpytor calls overlapping the planted deletion" "$(awk '{split($2, a, /[:-]/); if (a[1] == "chr20" && a[2] <= 453000 && a[3] >= 450001) n++} END {print n + 0}' calls.tsv)" 1 # GRIDSS (Java) looks up the container's hostname, which resolves only with a network. GRIDSS_IMAGE net ref,bam sh /smoke/gridss.sh at_least "records at the planted deletion" "$(sv_near gridss.vcf.gz)" 1 diff --git a/tests/smoke/delly_sex.sh b/tests/smoke/delly_sex.sh new file mode 100644 index 000000000..f07c31891 --- /dev/null +++ b/tests/smoke/delly_sex.sh @@ -0,0 +1,17 @@ +# shellcheck shell=sh +# DELLY_IMAGE row for the --sex option step 19 and the Nextflow DELLY pass. +# xmini.bam is mini.bam moved onto a chrX of GRCh38's length, past PAR1, so +# the planted deletion has no reads on a male chrX. Delly applies --sex only +# there, and ignores it without a word on a chrX of any other length: the +# genotype must be haploid (1) for a male and 1/1 for a female, and the log +# must name the sex. A value Delly does not know must fail the call. +set -e +for sex in male female; do + delly sr --sex "$sex" -g /in/xmini.fa -o "${sex}.bcf" /in/xmini.bam > "${sex}.log" 2>&1 || { cat "${sex}.log"; exit 1; } + grep -i 'sex=' "${sex}.log" || true +done +if delly sr --sex bogus -g /in/xmini.fa -o bogus.bcf /in/xmini.bam > bogus.log 2>&1; then + echo accepted > bogus.txt +else + echo rejected > bogus.txt +fi From 70ff52045941af73e86b8c821c74737cda52abfb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:32:25 +0100 Subject: [PATCH 4/5] docs(longread): say what step 04c checks and what was measured The long-read guide said duphold re-genotypes SVs and that Sniffles2's QUAL is reliable, and listed SV counts and repeat accuracy with no source. It now says what duphold really reads, describes the read-length check and the tandem-repeat map, and gives the run we measured on a public HG002 HiFi slice next to the Sniffles2 paper. testing.md says what the smoke rows and fake-docker cases cover. Bead: sv-17 Bead: sv-43 Bead: sv-41 --- CHANGELOG.md | 5 +++++ docs/long-read-guide.md | 25 ++++++++++++++++--------- docs/testing.md | 4 +++- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..500209b3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 04c (Sniffles2) now refuses a short-read BAM. Sniffles2 finds no SV in short reads and still exits 0, so the step reported success with an empty VCF. It now reads the mean length of the first 10,000 primary reads with the pinned samtools image and stops with exit 1 below 1,000 bp, naming the long-read BAM to use. Part of #411 (sv-18). + - `setup.sh` installs GIAB's GRCh38 tandem-repeat stratification (v3.6, `TANDEM_REPEATS_URL` and `TANDEM_REPEATS_SHA256`) and keeps it unpacked as `reference/GRCh38_AllTandemRepeats.bed`. Step 04c passes it to Sniffles2 as `--tandem-repeats`, which improves breakpoints and merging inside repeats, and says so when it is missing. On a public HG002 HiFi slice of chr20 it found one more benchmark SV (50 of 64 instead of 49) at the same precision. Sniffles' own GRCh38 file comes from PacBio's pbsv under a licence limited to PacBio data, so we use GIAB's. + - The step's closing hint no longer says duphold adds a GQ to long-read SVs. Sniffles2 writes its own QUAL and GQ, and step 15 reads Manta's calls only. The long-read guide's duphold row and its uncited accuracy lines are corrected (sv-17, sv-43). + - The SV smoke rows of Sniffles2, Manta, Delly and TIDDIT now need one PASS deletion of the planted size, not any record near it, and Sniffles2's row fails on a PASS call elsewhere. A new Delly row checks that `--sex` gives a haploid genotype on a male chrX of GRCh38's length (sv-41, sv-43). + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/long-read-guide.md b/docs/long-read-guide.md index ad37d77a0..f97909f2c 100644 --- a/docs/long-read-guide.md +++ b/docs/long-read-guide.md @@ -88,7 +88,7 @@ These tools are specifically designed for short-read data and will produce incor | 4a | TIDDIT | Short-read coverage/discordance model | Sniffles2 | | 4b | GRIDSS | Assembly-based, short-read specific | Sniffles2 | | 9 | ExpansionHunter | Illumina short-read graph model; expects paired-end data | TRGT (PacBio) or Straglr (ONT), outside this pipeline (vendor workflows) | -| 15 | duphold | Re-genotypes SVs using short-read depth models | Not needed — Sniffles2 QUAL scores are reliable | +| 15 | duphold | Adds depth fold-change tags (DHFFC, DHBFC) to Manta's calls and filters on them; reads Manta's VCF and the short-read BAM only | Not needed: Sniffles2 writes its own QUAL and GQ | | 18 | CNVpytor | Read-depth model calibrated for short reads | Sniffles2 detects CNVs natively | | 19 | Delly | Paired-end and split-read model | Sniffles2 | | 21 | Cyrius (CYP2D6) | Short-read depth-based star allele caller | Paraphase (long-read CYP2D6 resolver), outside this pipeline (vendor workflows) | @@ -217,11 +217,18 @@ ALIGN_DIR=aligned_longread ./scripts/04c-sniffles2.sh your_sample Output: `${GENOME_DIR}/your_sample/sv_sniffles/your_sample_sv.vcf.gz` -Sniffles2 excels at detecting all SV types from long reads. Typical output for a 30X long-read genome: -- 15,000-25,000 SVs (vs 7,000-9,000 from short-read Manta) -- Much better at insertions (short reads systematically miss these) -- Better breakpoint resolution (often single-nucleotide) -- Reliable in repetitive regions where short reads struggle +Before Sniffles2 runs, the step reads the first 10,000 primary reads of the BAM with the pinned samtools image. When their mean length is below 1,000 bp, it stops with exit 1 and names the long-read BAM to use. Sniffles2 finds no SV in short reads and still exits 0, so without this check a short-read BAM gave an empty VCF that looked like a finished run. Short-read SVs come from steps 04 (Manta), 04a, 04b and 19. + +`setup.sh` installs a tandem-repeat map for Sniffles2: every GRCh38 tandem repeat in GIAB's genome stratifications v3.6 (`reference/GRCh38_AllTandemRepeats.bed`, pinned by sha256 in `TANDEM_REPEATS_URL` and `TANDEM_REPEATS_SHA256`; NIST data, free to reuse). When the map is there, the step passes it as `--tandem-repeats`, which gives SVs inside repeats better breakpoints and merging. When it is not, the step prints a note and runs without it. We do not use the GRCh38 file that ships with Sniffles: it comes from PacBio's pbsv under a licence that allows PacBio data only. + +What we measured: the step on GIAB's public HG002 PacBio HiFi BAM (Sequel II, 15-20 kb, 10x), chr20:10-20 Mb, scored with truvari against GIAB's v5.0q SV benchmark inside its benchmark regions (chr20:10.1-19.9 Mb, PASS calls, 1 kb matching distance): + +| | Calls on chr20 | Recall | Precision | +|---|---|---|---| +| Without the tandem-repeat map | 62 | 0.77 (49 of 64) | 0.96 | +| With the map | 63 | 0.78 (50 of 64) | 0.96 | + +That is one 10 Mb window at low depth, not a genome-wide benchmark. For Sniffles2's own accuracy figures, see Smolka et al., Nature Biotechnology 2024 (https://doi.org/10.1038/s41587-023-02024-y). No CI case runs this step on real long reads; [Testing](testing.md) lists what checks it. ### Step 5: Continue with Standard Pipeline Steps @@ -346,10 +353,10 @@ The Docker image `CLAIR3_IMAGE` bundles models at `/opt/models/`. If you get a m 2. Check available models: `docker run --rm "${CLAIR3_IMAGE}" ls /opt/models/` (after `source versions.env` in the repository root) 3. If your ONT chemistry is different (e.g., R9.4.1), look for a matching model in the container -### "Sniffles2 produces 0 SVs" +### "Sniffles2 needs long reads" or "Sniffles2 produces 0 SVs" -- Verify the BAM is from long-read alignment (not short-read): `samtools stats your.bam | grep "average length"` -- Average read length should be >1000bp for ONT or >5000bp for HiFi +- Step 04c stops when the first 10,000 primary reads of the BAM average below 1,000 bp. Point `ALIGN_DIR` at the long-read BAM (step 02b writes `aligned_longread/_sorted.bam`). +- A mean above that but still low (below about 5,000 bp for HiFi) usually means a filtered or fragmented run: check `samtools stats your.bam | grep "average length"` - Check that the BAM is sorted and indexed ### "DeepVariant long-read model fails" diff --git a/docs/testing.md b/docs/testing.md index 7302f594c..8299d2918 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -173,7 +173,7 @@ This is every step script that no e2e case runs, plus the full-size index build | 01 (ORA to FASTQ) | needs Illumina's `orad` binary, which runs outside Docker | nothing | | 02b (long-read alignment) | long reads; the e2e fixture has none | the minimap2 image's smoke test, which maps synthetic HiFi reads with `-x map-hifi`; nothing runs the script | | 03e (Clair3) | long reads; the e2e fixture has none | the image's smoke test, with a recall check on synthetic HiFi reads and a check that the two model directories the script names exist; the fake-docker suite runs the script | -| 04c (Sniffles2) | long reads; the e2e fixture has none | the image's smoke test on the synthetic HiFi alignment; `tests/fake-docker/bash-step-partial-outputs.sh` runs the script | +| 04c (Sniffles2) | long reads; the e2e fixture has none | the image's two smoke rows on the synthetic HiFi alignment: exactly one PASS deletion of 2.7-3.3 kb at the planted one and no PASS call elsewhere, once as before and once with `--tandem-repeats`; `tests/fake-docker/bash-step-sv.sh` runs the script (a short-read BAM stops it before Sniffles2, and the tandem-repeat map is passed when installed and noted when not), and `tests/fake-docker/bash-step-partial-outputs.sh` reruns it after a failed run. No case runs it on real long reads; [the long-read guide](long-read-guide.md#step-4-structural-variant-calling) has a measured run on a public HG002 HiFi slice | | 15 (duphold) | the bash step: no case yet | case `nf-wiring-2-sv-str-tools` runs the Nextflow `DUPHOLD` and `DUPHOLD_FILTER` with the real image on the fixture's Manta calls; the image's smoke test on a planted deletion; `tests/fake-docker/bash-step-sv.sh` runs the script | | 18 (CNVpytor) | the bash step: no case yet | case `nf-wiring-2-sv-str-tools` runs the Nextflow `CNVPYTOR` and `CNVPYTOR_VCF` with the real image and the pinned GC and mask files; the image's smoke test; `tests/fake-docker/followups-ci-2-steps-03c-18.sh` runs the script (resource checks, mounts, commands); the SV merge case reads a planted CNVpytor VCF, not one CNVpytor wrote | | 29 (somatic Mutect2) | no case yet | `tests/fake-docker/bash-step-somatic-mito.sh` checks the Mutect2 command line | @@ -182,6 +182,8 @@ This is every step script that no e2e case runs, plus the full-size index build | 04a (TIDDIT) | no case yet | the image's smoke test; settle-doubts question 4 runs the script with only a BWA-MEM2 index; case `bash-step-bash42` and `tests/fake-docker/bash-step-sv.sh` run it against the fake docker | | chip-to-vcf (array data to VCF) on a vendor download | case `consumer-export-1` runs it only on the 23andMe and AncestryDNA files step 40 wrote | `tests/test_chip_to_vcf.sh` in the unit-tests job runs the script with the real bcftools and Picard images on two ten-row vendor files | +The SV callers' smoke rows (Sniffles2, Manta, Delly and TIDDIT) each need exactly one PASS deletion of 2.7-3.3 kb at the 3 kb deletion planted in the mini reference: a record of another type, size or filter there does not count. GRIDSS writes breakends, not deletion records, so its row still counts any record there. A second Delly row checks the `--sex` value that step 19 and the Nextflow `DELLY` process pass: it moves `mini.bam` onto a chrX of GRCh38's length past PAR1, where the deletion must come out haploid (`1`) with `--sex male` and `1/1` with `--sex female`, Delly's log must name the sex, and an unknown value must fail. Delly ignores `--sex` without a message on a chrX of any other length; the row goes red when that chrX is one base longer. `image-smoke.sh --self-test` plants records for each rule of these checks. + Case `nf-wiring-2-sv-str-tools` runs the Nextflow `EXPANSION_HUNTER` module with the real image on the fixture's BAM (the bundled catalog cut to the fixture's contigs); the bash step 09 runs in case `bash-step-09-str`. ## The settle-doubts job From 1ea64d40f7ae7fd39cd5b3e0d623d334639e6e4a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:07:05 +0100 Subject: [PATCH 5/5] docs(changelog): keep tracker ids out of the long-read entry --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 500209b3d..07f17157e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 04c (Sniffles2) now refuses a short-read BAM. Sniffles2 finds no SV in short reads and still exits 0, so the step reported success with an empty VCF. It now reads the mean length of the first 10,000 primary reads with the pinned samtools image and stops with exit 1 below 1,000 bp, naming the long-read BAM to use. Part of #411 (sv-18). +- Step 04c (Sniffles2) now refuses a short-read BAM. Sniffles2 finds no SV in short reads and still exits 0, so the step reported success with an empty VCF. It now reads the mean length of the first 10,000 primary reads with the pinned samtools image and stops with exit 1 below 1,000 bp, naming the long-read BAM to use. Part of #411. - `setup.sh` installs GIAB's GRCh38 tandem-repeat stratification (v3.6, `TANDEM_REPEATS_URL` and `TANDEM_REPEATS_SHA256`) and keeps it unpacked as `reference/GRCh38_AllTandemRepeats.bed`. Step 04c passes it to Sniffles2 as `--tandem-repeats`, which improves breakpoints and merging inside repeats, and says so when it is missing. On a public HG002 HiFi slice of chr20 it found one more benchmark SV (50 of 64 instead of 49) at the same precision. Sniffles' own GRCh38 file comes from PacBio's pbsv under a licence limited to PacBio data, so we use GIAB's. - - The step's closing hint no longer says duphold adds a GQ to long-read SVs. Sniffles2 writes its own QUAL and GQ, and step 15 reads Manta's calls only. The long-read guide's duphold row and its uncited accuracy lines are corrected (sv-17, sv-43). - - The SV smoke rows of Sniffles2, Manta, Delly and TIDDIT now need one PASS deletion of the planted size, not any record near it, and Sniffles2's row fails on a PASS call elsewhere. A new Delly row checks that `--sex` gives a haploid genotype on a male chrX of GRCh38's length (sv-41, sv-43). + - The step's closing hint no longer says duphold adds a GQ to long-read SVs. Sniffles2 writes its own QUAL and GQ, and step 15 reads Manta's calls only. The long-read guide's duphold row and its uncited accuracy lines are corrected. + - The SV smoke rows of Sniffles2, Manta, Delly and TIDDIT now need one PASS deletion of the planted size, not any record near it, and Sniffles2's row fails on a PASS call elsewhere. A new Delly row checks that `--sex` gives a haploid genotype on a male chrX of GRCh38's length. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped.