diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml index 2df8e31e8..144b3f886 100644 --- a/.github/workflows/freshness.yml +++ b/.github/workflows/freshness.yml @@ -23,6 +23,13 @@ name: Freshness # whole-tree link check runs monthly, so a link that dies upstream shows in the # issue instead of failing an unrelated pull request. # +# A manual run with pr_base runs the pull-request job too, as a pull request +# from those base commits would: the files changed and the URLs added are taken +# against each base, and the job checks the union. The landing train uses it to +# get this check on a branch's own tree before the pull request can run (a +# stacked branch conflicts with main until the branch before it lands). Each +# base must be a full sha and an ancestor of the ref. +# # 60-day rule: GitHub disables scheduled workflows in a public repository after # 60 days without repository activity. Monthly Dependabot and weekly Renovate merges # normally count as activity. As a fallback the keepalive job re-enables this @@ -40,6 +47,11 @@ on: # The 2nd at 05:17 UTC: after the monthly image test on the 1st. - cron: '17 5 2 * *' workflow_dispatch: + inputs: + pr_base: + description: 'Also run the pull-request job against these base commits (full shas separated by spaces, each an ancestor of the ref). Empty: a plain manual run.' + type: string + default: '' pull_request: branches: [main] @@ -142,7 +154,7 @@ jobs: run: scripts/ci/check-pgs-labels.sh --self-test pull-request: - if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.self-tests.result == 'success' }} + if: ${{ !cancelled() && (github.event_name == 'pull_request' || inputs.pr_base != '') && needs.self-tests.result == 'success' }} needs: self-tests runs-on: ubuntu-latest timeout-minutes: 20 @@ -154,9 +166,16 @@ jobs: - name: Files this pull request changes id: changed env: - BASE: ${{ github.event.pull_request.base.sha }} + EVENT: ${{ github.event_name }} + BASES: ${{ github.event.pull_request.base.sha || inputs.pr_base }} run: | - git diff --name-only "$BASE" HEAD > "${RUNNER_TEMP}/changed.txt" + if [ "$EVENT" != pull_request ]; then + for b in $BASES; do + [[ "$b" =~ ^[0-9a-f]{40}$ ]] || { echo "ERROR: pr_base takes full commit shas (the value is not echoed)"; exit 1; } + git merge-base --is-ancestor "$b" HEAD || { echo "ERROR: ${b} is not an ancestor of ${GITHUB_SHA}"; exit 1; } + done + fi + for b in $BASES; do git diff --name-only "$b" HEAD; done | awk '!seen[$0]++' > "${RUNNER_TEMP}/changed.txt" cat "${RUNNER_TEMP}/changed.txt" has() { grep -qxE "$1" "${RUNNER_TEMP}/changed.txt" && echo true || echo false; } { @@ -186,9 +205,12 @@ jobs: - name: Every download URL this pull request adds answers env: - BASE: ${{ github.event.pull_request.base.sha }} + BASES: ${{ github.event.pull_request.base.sha || inputs.pr_base }} OG: ${{ github.workspace }}/rust/target/release/og - run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY" + # The exit is inside the braces: a variable set left of a pipe is lost, so + # the failure has to leave as the left side's status (pipefail). + run: | + { rc=0; for b in $BASES; do scripts/ci/check-links.sh --since "$b" || rc=1; done; exit "$rc"; } | tee -a "$GITHUB_STEP_SUMMARY" - name: Every PGS ID's label matches the PGS Catalog if: steps.changed.outputs.prs == 'true' @@ -294,7 +316,7 @@ jobs: # Fallback for the 60-day rule described at the top of this file. No `needs`: # it runs even when an upstream outage fails the checks. keepalive: - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && inputs.pr_base == '' runs-on: ubuntu-latest timeout-minutes: 5 permissions: diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index f1a4c1821..1fbfdc628 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -348,6 +348,26 @@ jobs: [ "$rc" -ne 0 ] || { echo "ERROR: the planted unreadable-report success passed"; exit 1; } grep -E 'unreadable_pharmcat_is_unassessed.*FAILED' "$log" echo "Control: the unreadable-report success was reported as a failed test." + # The live ops card (og-live/tests/ops*.rs): each planted_* test turns + # on one planted mistake of the fault-inject feature, so every one of + # them must fail, and none may pass. + - name: Control (every planted mistake of the live ops card must fail its check) + run: | + set -o pipefail + tests=(--test ops --test ops_eta --test ops_headroom) + log="${RUNNER_TEMP}/ops-planted.log" + want=$(cargo test --locked -p og-live --features fault-inject "${tests[@]}" planted_ -- --list 2>/dev/null \ + | grep -cE '^planted::planted_[a-z0-9_]+: test$' || true) + [ "$want" -gt 0 ] || { echo "ERROR: no planted test was found"; exit 1; } + rc=0 + cargo test --locked --no-fail-fast -p og-live --features fault-inject "${tests[@]}" planted_ > "$log" 2>&1 || rc=$? + grep -E '^test planted::planted_' "$log" || true + [ "$rc" -ne 0 ] || { cat "$log"; echo "ERROR: every planted mistake passed"; exit 1; } + passed=$(grep -E '^test planted::planted_[a-z0-9_]+ \.\.\. ok$' "$log" || true) + [ -z "$passed" ] || { echo "ERROR: a check passed with its planted mistake on:"; echo "$passed"; exit 1; } + failed=$(grep -cE '^test planted::planted_[a-z0-9_]+ \.\.\. FAILED$' "$log" || true) + [ "$failed" -eq "$want" ] || { cat "$log"; echo "ERROR: ${want} planted tests, ${failed} failed"; exit 1; } + echo "Control: all ${failed} planted mistakes of the live ops card failed their checks." - name: Traits controls (diplotype order, REF check and missing phase rows) env: CARGO_BUILD_JOBS: 3 @@ -371,6 +391,37 @@ jobs: cp -f "$saved" "$src" trap - EXIT + - name: Control (ROH scope and neutral hints reject planted faults) + run: | + reader=crates/og-summary/src/collect/group_c/roh.rs + table=../assets/imprinted_regions.tsv + cp "$reader" "$RUNNER_TEMP/roh-reader.rs" + cp "$table" "$RUNNER_TEMP/imprinted-regions.tsv" + restore_roh() { + cp "$RUNNER_TEMP/roh-reader.rs" "$reader" + cp "$RUNNER_TEMP/imprinted-regions.tsv" "$table" + } + trap restore_roh EXIT + sed -i 's/let denominator = covered_unmasked_bp(&covered);/let denominator = autosome_unmasked_bp();/' "$reader" + sed -i 's/s("-")/s("Example syndrome")/g' "$reader" + sed -i 's/26248010;29959430/not-pmid/' "$table" + log="$RUNNER_TEMP/roh-scope-planted.log" + rc=0 + cargo test --locked -p og-summary --no-fail-fast > "$log" 2>&1 || rc=$? + cat "$log" + test "$rc" -ne 0 + for name in \ + roh_scope_chr20_uses_its_own_unmasked_denominator \ + roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty \ + roh_scope_a_segment_only_file_never_establishes_scope \ + roh_scope_upd_is_a_region_hint_without_syndrome_names \ + imprinted_asset_is_the_only_table; do + grep -E "test .*${name} .* FAILED" "$log" + done + grep -E 'test result: FAILED\. [0-9]+ passed; 5 failed;' "$log" + restore_roh + trap - EXIT + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7c64de319..e8b0c09a4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -132,7 +132,7 @@ jobs: run: | echo "Checking internal markdown links..." rm -f /tmp/link_fail - for md in docs/*.md README.md; do + for md in docs/*.md docs/results/*.md README.md; do # Extract markdown links: [text](target) grep -oP '\]\([^)]+\)' "$md" 2>/dev/null | \ grep -oP '\(([^)]+)\)' | tr -d '()' | \ diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index e40adeb06..4240b63e6 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -8,7 +8,10 @@ name: Public evidence # downloaded once (its md5 checked) and indexed, for # every chromosome job. # archaic-hg002 setup.sh --archaic and step 43 on HG002's GIAB v4.2.1 -# genotypes, one job per autosome of the chromosomes +# genotypes (with a gVCF of one reference block per +# GIAB benchmark interval, since step 43 reads a site +# with no record as the reference only inside one), +# one job per autosome of the chromosomes # input (all 22 by default: about 150 GB of archaic # downloads in all; each job fetches its three files # side by side first, and setup.sh resumes them and @@ -22,7 +25,9 @@ name: Public evidence # list must make it fail. # archaic-same-person the method check: one 1000 Genomes person with # Chen et al.'s released calls, step 43 on her high- and -# low-coverage genotypes, beside the released figure. +# low-coverage genotypes (each with a gVCF of one +# reference block per chromosome), beside the released +# figure. # ancient-footprint setup.sh --ancient-panel on the full AADR 1240K files, # then step 44 on HG002, each with the peak memory of # its containers (the cgroup's memory.peak), the peak @@ -216,6 +221,28 @@ jobs: mkdir -p "${G}/HG002/vcf" curl -fsSL --retry 5 -o "${G}/HG002/vcf/HG002.vcf.gz" "${GIAB_HG002}.vcf.gz" curl -fsSL --retry 5 -o "${G}/HG002/vcf/HG002.vcf.gz.tbi" "${GIAB_HG002}.vcf.gz.tbi" + # Step 43 reads a site with no record as the reference only inside a + # reference block of the gVCF beside the VCF. GIAB states HG002 + # matches the reference inside its benchmark regions and makes no + # claim outside them, so the gVCF is one block per benchmark + # interval of the part's chromosomes (the VCF gives the variants). + curl -fsSL --retry 5 -o "${EV}/truth.bed" "${GIAB_HG002}_noinconsistent.bed" + FAI="${G}/reference/GRCh38_no_alt_analysis_set.fasta.fai" + { + printf '##fileformat=VCFv4.2\n' + for c in $CHROMS; do awk -F'\t' -v c="chr${c}" '$1 == c {printf "##contig=\n", $1, $2}' "$FAI"; done + printf '##ALT=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHG002\n' + for c in $CHROMS; do + awk -F'\t' -v c="chr${c}" 'BEGIN {OFS = "\t"} $1 == c {print c, $2 + 1, ".", "N", "<*>", ".", ".", "END=" $3, "GT", "0/0"}' \ + "${EV}/truth.bed" | sort -k2,2n + done + } > "${EV}/HG002.g.vcf" + docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ + bcftools view -Oz -o /g/HG002/vcf/HG002.g.vcf.gz /e/HG002.g.vcf + echo "gVCF: $(grep -vc '^#' "${EV}/HG002.g.vcf") reference blocks" for pop in ALL EUR; do if ! ARCHAIC_POP=$pop GENOME_DIR="$G" scripts/43-archaic.sh HG002 2>&1 | tee "${OUT}/step43-${pop}.log"; then echo "step 43 (${pop}) failed"; exit 1 @@ -359,9 +386,12 @@ jobs: set -euo pipefail . ./versions.env G="${EV}/genome" OUT="${EV}/out" + DATA="${G}/reference/archaic/${ARCHAIC_DATA_VERSION}" mkdir -p "$OUT" - # sub LABEL URL-PREFIX NAME-OF-FILE-PATTERN: the person's non-reference - # SNVs on the chromosomes, one VCF. + # pull LABEL URL-PREFIX NAME-OF-FILE-PATTERN: the person's SNV records + # on the chromosomes that are not homozygous reference, one VCF: the + # ALT calls, and the sites with no call, which step 43 reads as + # unknown (a VCF record wins over the gVCF's reference block). pull() { local label=$1 url=$2 pat=$3 c parts="" mkdir -p "${G}/${SAMPLE}${label}/vcf" @@ -370,8 +400,8 @@ jobs: # The container has no CA bundle, so curl fetches the file here. curl -fsSL --retry 5 -o "${EV}/in.vcf.gz" "${url}/${f}" curl -fsSL --retry 5 -o "${EV}/in.vcf.gz.tbi" "${url}/${f}.tbi" - docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" "$BCFTOOLS_IMAGE" \ - bcftools view -s "$SAMPLE" -v snps --min-ac 1 -Oz -o "/e/${label}.raw.vcf.gz" /e/in.vcf.gz + docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' + bcftools view -s "$1" -v snps -Ou /e/in.vcf.gz | bcftools view -e "GT=\"ref\"" -Oz -o "/e/$2.raw.vcf.gz"' _ "$SAMPLE" "$label" # The 2019 recall names the contigs 22, not chr22: rename them # (a name already chrN stays as it is). printf '%s\tchr%s\n' "$c" "$c" > "${EV}/rename.txt" @@ -387,8 +417,35 @@ jobs: docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools index -t -f "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" rm -f "${EV}/${label}".chr*.vcf.gz - echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records" + # How many records, how many of them with no call, and how many of + # those at a table site (the sites step 43 reads as unknown). + docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools query -i 'GT="mis"' -f '%CHROM\t%POS\n' \ + "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" > "${EV}/${label}.nocall.tsv" + echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records," \ + "$(wc -l < "${EV}/${label}.nocall.tsv") of them with no call," \ + "$(for t in "${DATA}"/chr*.tsv.gz; do gzip -dc "$t"; done \ + | awk -F'\t' 'FILENAME == ARGV[1] {m[$1 FS $2] = 1; next} ($1 FS $2) in m {n++} END {print n + 0}' "${EV}/${label}.nocall.tsv" -) of those at a table site" + # Step 43 reads a site with no record as the reference only inside + # a reference block of the gVCF. A panel VCF holds no callable mask, + # so the gVCF is one block over each whole chromosome: a site with + # no record (the person is 0/0 there, or the panel has no variant + # there) reads as the reference, as this check measured before step + # 43 needed a gVCF, and a site with no call stays unknown. + { + printf '##fileformat=VCFv4.2\n' + for c in $LIST; do awk -F'\t' -v c="chr${c}" '$1 == c {printf "##contig=\n", $1, $2}' "$FAI"; done + printf '##ALT=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "${SAMPLE}${label}" + for c in $LIST; do + awk -F'\t' -v c="chr${c}" 'BEGIN {OFS = "\t"} $1 == c {print c, 1, ".", "N", "<*>", ".", ".", "END=" $2, "GT", "0/0"}' "$FAI" + done + } > "${EV}/${label}.g.vcf" + docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ + bcftools view -Oz -o "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.g.vcf.gz" "/e/${label}.g.vcf" } + FAI="${G}/reference/GRCh38_no_alt_analysis_set.fasta.fai" pull hc "$HC_URL" '1kGP_high_coverage_Illumina.chr@.filtered.SNV_INDEL_SV_phased_panel.vcf.gz' pull lc "$LC_URL" 'ALL.chr@.shapeit2_integrated_snvindels_v2a_27022019.GRCh38.phased.vcf.gz' args=() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 936471704..c8c3b385c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,8 @@ on: required: true type: string # A dry run: a pull request that changes this file or rust/ builds the og - # binaries and their SHA256SUMS as an artifact; nothing is released. + # binaries, THIRD-PARTY-LICENSES.txt and their SHA256SUMS as an artifact; + # nothing is released. pull_request: branches: [main] paths: ['.github/workflows/release.yml', 'rust/**'] @@ -21,7 +22,9 @@ permissions: jobs: # The og binaries every release carries (docs/rust.md): both musl builds # and the macOS pair (without BAM, for a live-site host), made by - # scripts/setup.sh --og as a checkout makes them, with SHA256SUMS. + # scripts/setup.sh --og as a checkout makes them, with + # THIRD-PARTY-LICENSES.txt (the notices of every crate and C library they + # link) and SHA256SUMS over all five. # A checkout of the tag downloads them instead of building (rust/og.sh). og-binaries: runs-on: ubuntu-latest @@ -44,16 +47,17 @@ jobs: ref: ${{ inputs.tag || github.ref }} persist-credentials: false - - name: Build og for Linux and macOS (scripts/setup.sh --og builds in OG_BUILDER) + - name: Build og for Linux and macOS, and its licence notices (scripts/setup.sh --og builds in OG_BUILDER) run: | # On a tag, setup.sh --og would download the release's own binaries, # which this job is about to make: build them. rust/og.sh build rust/og.sh check --required mkdir -p dist - cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 bin/.og/og-darwin-arm64 bin/.og/og-darwin-x86_64 dist/ + cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 bin/.og/og-darwin-arm64 bin/.og/og-darwin-x86_64 bin/.og/THIRD-PARTY-LICENSES.txt dist/ file dist/og-* - (cd dist && sha256sum og-* > SHA256SUMS && cat SHA256SUMS) + (cd dist && sha256sum og-* THIRD-PARTY-LICENSES.txt > SHA256SUMS) + scripts/ci/check-release.sh dist dist - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -108,7 +112,7 @@ jobs: echo "" echo "## og binaries" echo "" - echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. og-darwin-arm64 and og-darwin-x86_64 run the live site's genome map on a Mac; they leave out the tools that read BAMs. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks all four." + echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. THIRD-PARTY-LICENSES.txt holds the licences of the Rust standard library, the crates and the C libraries (musl, htslib, htscodecs) the binaries link. og-darwin-arm64 and og-darwin-x86_64 run the live site's genome map on a Mac; they leave out the tools that read BAMs. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks all four." } >> "${RUNNER_TEMP}/notes.md" # Only a commit that main's CI passed is released: it must be on main, @@ -135,10 +139,12 @@ jobs: # rust/og.sh install checks a downloaded binary with # `gh attestation verify` when gh is installed. - - name: Attest the og binaries + - name: Attest the og binaries and their licence notices uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - subject-path: 'dist/og-*' + subject-path: | + dist/og-* + dist/THIRD-PARTY-LICENSES.txt - name: Create GitHub Release uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 @@ -152,4 +158,5 @@ jobs: dist/og-linux-aarch64 dist/og-darwin-arm64 dist/og-darwin-x86_64 + dist/THIRD-PARTY-LICENSES.txt dist/SHA256SUMS diff --git a/.gitignore b/.gitignore index 7f135ae3a..c9d40d5df 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,8 @@ work/ .nextflow/ .nextflow.log* results/ +# docs/results/ is the results guide, one page per report chapter, not run output +!docs/results/ results_*/ .nf-test/ /nextflow diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e6e8b1d8..ae41486ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,24 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- ROH summaries now divide F_ROH by the GRCh38 length outside the bands of the assessed autosomes. Per-site rows identify those chromosomes, including chromosomes with no ROH. A new `roh.data.scope` records the chromosomes, denominator and scope source. Relationship labels and UPD flags require sites on all 22 autosomes and the full unmasked autosome length. Segment-only files use their observed chromosomes as an unverified lower bound and get neither label. Empty input has null F_ROH. Segment counts and the step 11 summary table keep their format. +- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. The container builder mounts it, and the source stamp includes its blob hash, so a table edit makes an old binary stale. The stamp format and `tree-hash` command name stay the same. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. +- The ROH collector now streams per-site rows instead of keeping the whole file in memory. On 1.1 million synthetic site rows, peak RSS fell from 65.7-67.0 MB to 3.0-3.5 MB. CPU time stayed near 0.2 seconds. Under shared host load, wall time ranged from 1.40-21.46 seconds before and 7.89-38.16 seconds after; these timings do not establish a speedup. +- Step 23's ClinVar tier now holds only the allele ClinVar classified. It took every VEP record at a step 6 hit's position, so when a call of T>A,C split into two records and ClinVar classified only T>A, the T>C record also landed in `clinical/_clinvar_pathogenic.vcf.gz`, in the tier's count and, when no other tier kept it, in `_clinical.vcf.gz` and `_clinical_summary.tsv` with `FREQ_TIER` read as a ClinVar record. The tier now matches CHROM, POS, REF and ALT. A combined hit of step 6 (two neighbouring records that make one allele, `PGP_JOINED_FROM`) was often lost from the tier, because its left-aligned position matches neither part; both parts now enter the tier, and its count counts them as two records. No file, column or option changes; only which records these files hold. The reports' ClinVar section reads the step 6 hits and does not change. The Nextflow `CLINICAL_FILTER` module takes the tier from each record's own ClinVar annotation and is not affected. +- A heteroplasmic MT-RNR1 variant now gives the aminoglycoside warning. Step 07 and the Nextflow PHARMCAT process passed a named MT-RNR1 allele to PharmCAT only at an allele fraction (AF) of 0.95 or more, so m.1555A>G or m.1494C>T at AF 0.05 to 0.95 left MT-RNR1 No Result and the warning was lost. CPIC sets no heteroplasmy level at which aminoglycosides are safe, so `og pgx mito-merge` now passes a PASS named allele from AF 0.05, the pipeline's heteroplasmy floor, and its log says `heteroplasmic` with the AF. Below AF 0.05 the allele is not passed, the log says it is below the detection floor, and the gene is not called Reference either; beside a named allele at the floor it is only named in the log and does not withhold that call. The default of `--min-af` moves from 0.95 to 0.05; the option stays. A named allele whose AF reads `nan` is now held back as having no allele fraction; it was passed as homoplasmic. No output file or format changes: the MT-RNR1 line of the outside calls now appears in these cases. Part of #411 (pgx-04, pgx-32). + +- `run-all.sh` now sizes Nextflow from the container engine, not only from the host. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac, but `run-all.sh` gave Nextflow the Mac's CPUs and RAM. Nextflow then started more tasks than the VM could hold, and the VM killed them (exit 137). When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` now passes the engine's figures as the default `--max_cpus` and `--max_memory`. It also writes them to `/nextflow/executor.config` and passes that file with `-c`, so Nextflow books no more than the VM has. A `-c` of your own comes after it, so each `executor` setting in it wins. `THREADS`, `--max_cpus` or `--max_memory` of your own still cap each task, and `executor.config` never books less than them, so Nextflow does not refuse a task that asks for more than the VM's figure. On native Linux the engine reports the host's figures, and the command is unchanged. `validate-setup.sh` adds an `[INFO]` line when Docker has less memory than the host. No option or output file changes. +- `benchmark-variants.sh` now compares genotypes, not just alleles, between callers. Its pairwise mode counted a site as shared when both callers had the same allele there, so a site one caller called 0/1 and the other 1/1 still scored as full agreement, and the summary said a Jaccard index of 1.0 meant perfect agreement. `comparison.tsv` gains two columns at the end, `GT_Concordant` and `GT_Concordance`: the shared sites whose genotypes also agree, ignoring phase and allele order, with a homozygous call equal to its haploid form; a site with no genotype on either side is not a match. The existing columns are unchanged. + - The step now also compares Clair3 (`vcf_clair3/`) and `.vcf.gz` in any other `vcf_*/` directory. `run-all.sh` with `BENCHMARK=true` already queued the step for those directories, and the step then found one caller and exited 1, so the run reported a failure; step 03e also told long-read users to run it. + - The docs stop calling step 03a the clinical-lab standard without a caveat: it runs HaplotypeCaller without base quality score recalibration (BQSR) and with `-ERC NONE`, so it is not the full GATK Best Practices workflow. Part of #411 (plat-55, plat-71). + +- Step 42 no longer reads a trait from a low-quality call. It kept each site's GQ but never checked it, so a call of one or two reads (GQ 0) at APOE gave a confident e2/e4 or e4/e4, and the same held for HFE, SERPINA1 and every row of the traits table; a call the caller had filtered (FILTER `LowQual`) was read as well. A call with a GQ below 20, or with a FILTER other than `PASS`, `.` or DeepVariant's `RefCall`, now has the new status `low_gq`: the row keeps its genotype, gives no reading, and says "not read; check the reads" with the GQ or the filter. A rule with such a site reads `low_gq` with no result, and such a site is not sent to WhatsHap. GQ 20 and a call without a GQ are read as before. The floor of 20 is the one `og pgx refblock-fill` and the Y private SNPs use. What changes in the outputs: `traits/_traits_genotypes.tsv` gains a seventh column, the record's FILTER; `low_gq` is a new value of `status` in `traits/_traits.tsv` and `traits/_genotype_rules.tsv`; summary.json's traits section gains a `low_gq` count beside `called`, `not_covered` and `other_allele`. No column is renamed or removed. The APOL1 G2 deletion that `og summary collect` reads beside the rule takes the same floor: a call over its span below GQ 20, or a reference call with a FILTER other than `PASS`, `.` or `RefCall`, leaves G2 not read, so two G2 copies at GQ 5 no longer read G2/G2. +- E2E base artifacts exclude the Cyrius installation and pypgx bundle. The finished archive is checked before upload, with planted failures in the runner self-test. The shards still receive pgsc_calc and the tools' results. No pipeline output changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. +- The always-report list of step 06 (`clinvar/_clinvar_deep_intronic.tsv`) now says what each call really is. A `0/0` with a filter such as `LowQual`, a `0/0` without a GQ, and a `0/0` read from one or two reads all read as plain `ref`, and a male's one-copy chrX call (`1` or `0`, as some callers write it) read as the raw `1` or as no call. One state table in `og clinvar deep-intronic` now decides each site. Only FILTER `PASS`, `.` and `RefCall` leave a reference call plain; any other FILTER makes it `ref (low confidence)` and names the FILTER. A reference call without a GQ is `ref (low confidence)` too, and so is one below GQ 20 when `CLINVAR_MIN_GQ` is set lower than 20: a lower floor changes the hits only. A depth below the step's depth floor (`CLINVAR_MIN_DP`, `ext.min_dp`, default 10, passed with the new `--min-dp`) reads `ref (shallow)`. On chrX and chrY outside the pseudoautosomal regions a one-copy `1` reads `hemizygous` and a one-copy `0` reads `ref`; a one-copy GT anywhere else is no call, with why. A no call says what the record holds (`has no genotype`, `is not a reference call`), and no longer claims a PASS check that never ran or that no record exists when a VCF record without a genotype, or a record for another allele, does. Both mirrors of step 06 add FILTER as a ninth column of the query the reader takes. Output change: the file keeps its columns; the `call` column gains the words `ref (shallow)` and `hemizygous`, and some `detail` texts change. The parity goldens of `clinvar-deep-intronic` record the deliberate changes. +- Step 06 now counts the sample records whose REF is not the reference's base. `bcftools norm -c w` keeps such a record as written and only warns `REF_MISMATCH` in the log, and the record cannot match a ClinVar allele. The step's summary, and the Nextflow module's log, now print the count, and a notice to check the VCF's reference when there are any. - Each ClinVar hit now keeps the condition ClinVar names for it. Step 06 copied only ClinVar's ID, gene, significance and review status onto each hit, so a later step knew the gene but not the disease: a RET hit could be Hirschsprung disease or MEN2. The hits VCF (`clinvar/_clinvar_hits.vcf`) now also carries `CLNDN`, ClinVar's disease name, and `CLNDISDB`, its disease database ids, in both the bash step and the Nextflow module. Two INFO fields are added; nothing is renamed or removed, and `clinvar/_clinvar_hits.tsv` is unchanged. A hand-built ClinVar file that does not define the two fields still runs, without them and with a notice. - A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed (`1/.` and `./1`, the two alleles of a split `1/2` site, count as both copies), or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. - CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. An older or external CRAM written without stored MD/NM gets them regenerated on restore, as before. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. @@ -16,6 +31,31 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. +- Each release now ships `THIRD-PARTY-LICENSES.txt` beside the og binaries. The binaries link about seventy crates, the Rust standard library and the C libraries musl, htslib and htscodecs statically, and their licences (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their notices; no release carried any. `rust/og.sh build` writes the file with the new `og dev licences notices`, from `cargo metadata` for each of the four targets and the licence files each crate ships, including htslib's and htscodecs' own from the `hts-sys` crate, plus the notices of the Rust standard library and of musl, the C library of the Linux pair. The build stops when a linked crate's licence is outside the allow list in `docs/rust.md`. The release adds the file to `SHA256SUMS`, which `sha256sum og-*` alone left out, and to the build attestation, and `scripts/ci/check-release.sh dist` refuses a release whose files or sums do not match. No pipeline output changes. +- The live site's run health card and its `/metrics` text now have a Rust port: og-live's ops module computes the hours left, the memory and disk headroom, the last activity and the five alerts from og-runlog, the same code `og run eta` and `og run headroom` use. It writes the same card and the same metrics text as `bin/owngenome_live_ops.py`, which the Python site keeps running until the Rust site replaces it. A parity test runs both on the same inputs: the card must match by meaning, the metrics text byte for byte. On the made-up run of `tests/fixtures/live/ops`, one card takes 0.14 ms and 4.7 MB peak memory, against 0.55 ms and 15 MB for the Python plugin. + - `tests/test_owngenome_live_ops.py`, `tests/test_run_eta.py` and `tests/test_run_headroom.py` are now cargo tests in `rust/crates/og-live/tests/`. Their planted source mistakes are named faults of a new `fault-inject` cargo feature of og-runlog and og-live, and Guard's rust job requires every one of them to fail its check. The release binary holds none of them. + - One behaviour differs on purpose: a step history the reader cannot use, such as one with no `trace_file` column, no longer turns the card into "run health could not be read". The card still shows; the hours left then come from the catalogue's hours. +- The guide to interpreting results now has one page per report chapter, under `docs/results/`: secondary findings and carrier status, ClinVar and variant annotation, pharmacogenomics and HLA, structural variants and repeats, ancestry and lineage, telomeres and somatic variants, and data quality. The text moved word for word. `docs/interpreting-results.md` keeps the "Before You Look" bullets, the "Before You Panic" table, the example outputs and the general advice, and lists every chapter page. Each old link to a section of that page still works: it lands on a link to the section's new place. + - The repeat table moved with its chapter, so `tests/test_str_ranges.py` now reads it from `docs/results/structural-and-repeats.md`. + - Two moved lines were wrong and are corrected. The clinical filter paragraph said step 23 keeps every HIGH-impact variant with no frequency filter; step 23 keeps HIGH and MODERATE variants below 1% `MAX_AF`, those of unknown frequency in their own tier, and ClinVar pathogenic ones at any frequency. The CYP2D6 row said poor metabolizers get a toxic buildup of codeine, tramadol and tamoxifen; for those three a poor metabolizer gets less effect, and an ultrarapid metabolizer risks opioid toxicity. + +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: + - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. + - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus until you move inside the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Step 41 (blood groups) reads every VCF record that covers a marker site, and their order no longer changes the answer. Before, the first record at a position won. An all-sites VCF writes a reference record and then the indel at one position (`T . 0/0`, then `T TC 0/1` at ABO c.261), so a person with an A or B allele read O. A split multi-allelic record with the SNV first did the same. No column, file or option changes; only the values of the affected sites do. What changes: + - A call of the marker's ALT now wins over the other records at the position. Another allele a second record calls there is named in the note, as when one record holds both. + - Distinct ALTs listed by one null marker add their copies across split records. JK_01N06 and KEL_02N01 now give the same evidence and null/null phenotype before and after splitting. Repeated calls of the same ALT count once. Conflicting dosages for one ALT, or totals above two copies, give a no call with an incompatible-dosage note. + - A no call of a record that names the marker's ALT (`./.`, or a failed FILTER) now wins over a reference record at the same position, so the site is a no call instead of a confident reference. + - A 0/0 record over a REF shorter than the marker's (an all-sites `C . 0/0`, or a gVCF block that starts at the position) now counts as no record, so the reads decide. Before it was a no call: ABO read "A, A1 or A2 not determined", and Vel, HPA-14 and every system with a deletion marker lost confidence. + - A record that starts upstream and spans a marker with an allele that changes the marker's base (a deletion over it, or a padded `CG CA`, as DeepVariant writes `CG C,CA` when a deletion and an SNV overlap) now makes the site a no call, with a note that names the record. Before, the site had no record at its own position and 8 reads made it a confident reference: a K/K person read k/k at high confidence. + - A VCF with one record per position and no overlapping records, as DeepVariant writes, gives the same output as before: the 34 parity cases still agree with the Python oracle byte for byte. +- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many of the looked-up items failed and that the check is incomplete, or "Could not check: all N looked-up items failed" when none could be checked. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, credits the allocated space of a partial download, and warns without stopping the remaining installs. Stale extraction files are removed before the disk check. Retained PCGR archives are checked against the pinned digest before reuse. `--clinical-data` exits 1 when one item is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. +- Step 43 (archaic ancestry) no longer counts a site without reads as a match with the reference genome. It wrote every table site the VCF had no record for as 0/0, so a stretch with no reads, where the archaic genomes carry the common reference allele, could score as a long Neanderthal segment, and one call anywhere on a chromosome was enough to analyse all of it. On a synthetic chromosome with one call far away and no reads over 150 table sites, the pinned IBDmix called a 149 kb segment and a 100% Neanderthal share; it now calls none. `og archaic genotypes` takes the sample's gVCF (`--gvcf`) and reads such a site as the reference only inside a confident reference block: FILTER PASS or `.`, every allele of GT 0, and GQ and depth (MIN_DP, else DP) of 1 or more where given. Any other site with no record goes to IBDmix as unknown (9). The analysed span and `analysed_sites` now run over the sites read, and the command prints how many sites it read and how many it did not. Step 43 now needs the gVCF of step 3, as steps 42 and 44 do: the bash step stops without it and says why, the Nextflow pipeline skips a samplesheet row without one, with a warning, and `run-all.sh` lists step 43 as `skipped (no gVCF: ...)` for a sample that starts from a VCF without one beside it. The `archaic-hg002` jobs of `public-evidence.yml` give HG002 a gVCF of one reference block per GIAB benchmark interval, and `archaic-same-person` gives one block over each whole chromosome of the 1000 Genomes panel person, so a site with no record reads as the reference, as that check did before, while the panel's sites with no call for the person are kept as records and stay unknown. Without `--gvcf`, a site with no record is unknown; a VCF with a record at every site still needs no gVCF, and its reference calls with ALT `.` are read as the reference (they were skipped before). A gVCF with no records on a table chromosome of the VCF, or with a record out of order anywhere, stops the step with exit 2. No option, output file or column is renamed or removed; the values of `_archaic.tsv` and the segments move for a sample with gaps in its coverage. The parity goldens of `archaic-genotypes` record the change. `docs/43-archaic.md` now also says how the 100 stand-ins carry round(AF × 200) ALT alleles with your own genotype counted in the frequency, as Chen et al. counted each person in their group's, what the tests check and what they do not, and the terms of the MPI-EVA archaic genomes, which the pipeline does not redistribute. +- Step 41 (blood groups) now matches an indel however the caller wrote it. The marker table writes each indel left-aligned, and the step matched records by position, so the same deletion written further right in its repeat missed the marker. With reads there the site then read as the reference allele at confidence high: ABO c.1061delC written as `GG G` at chr9:133255671 gave A1/A1 instead of A2/A2, and a Kell-null deletion written that way gave k/k instead of null/null. The step now left-aligns the VCF's records within 100 bp of each marker site against the reference first (`bcftools norm -c w -f`; in the pipeline the `BLOODTYPE` process does it, and now runs in `BCFTOOLS_IMAGE`), and reads that file. Multi-allelic records stay whole, so a `C A,G 1/2` at ABO c.803 keeps its one B copy. Records on a contig the reference lacks go through unchanged. Step 41 now needs the reference and its `.fai` (`REF_FASTA`). The output files and their columns do not change; a sample whose VCF had such a right-shifted marker indel now gets that allele called. +- `og ancestry-fine simulate` builds an admixed genome with known truth, for testing the fine-ancestry engines that replace ADMIXTURE and RFMix in step 26b. It copies haplotypes of held-out panel people: ancestry tracts come at T events per Morgan with each ancestry drawn from the given proportions, and inside a tract the copied haplotype switches at 1 per Morgan by default. It writes an unphased VCF (sorted alleles, so nothing says which copy an allele came from) and a truth BED of each copy's ancestry that covers every chromosome from its first site to its last with no gap or overlap. The same seed gives the same bytes. T and the switch rate are at most 10,000 per Morgan: far above that, the draw of events never ends. On a synthetic panel of 1,000 people and 100,000 sites it takes about 1 s and 13 to 14 MB. No step calls it, and no output of a run changes. The new crate `og-admix` holds it and what the engines will share: the panel reader (phased biallelic SNVs, bit-packed), the genetic map, the `model/ref.geno` file of packed reference genotypes and a seeded xoshiro256** generator. `og ancestry-fine chroms` now reads its maps with this reader. What changes on purpose: + - A map with a cM value that is not finite, or with cM going down as the position goes up, now stops `og ancestry-fine chroms` with an error naming the file. Before, a `nan` cM dropped the chromosome without a word, and a decreasing map gave a wrong span. Beagle's GRCh38 maps never go down. At a first position that holds two markers, the later marker's cM now counts, as it already did at every other position. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. @@ -211,7 +251,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. +- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order. Two things differ for them. Another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag; the family page reads "Report closed" for that child instead, and shows the count `family.json` gave before once it opens the child's results. And the home tile of a child with no result to act on now, whose `summary.json` is older than a finished step and still names one, counts from the summary its `results.json` reads. A person any family lists as a child is closed on their page, the home page and every family page that lists them, whatever the other families say; a family where they are a partner keeps that role, with their buttons under its members. Whether a page keeps results closed now comes from the server's answer in `state.json` alone: an adult whose family's `family.json` has not answered yet, or fails, gets their normal page, where before such a page kept their results to act on closed until opened. One deliberate difference shows only while a file of a family folder cannot be read: the people that family may list are treated as children, closed on their pages, the home page, the family page and the finding addresses, until the file reads again (everyone while its members cannot be read, or while its members file names no one; its own members while another of its files cannot be read). The server works this out from the files on each request. APOE keeps its own warning. These outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child and a new `child` field for each member run; `family.json` answers 500 with a fixed line when a family file cannot be read, where it dropped the connection; `/runs.json` counts a child's results from the summary its `results.json` reads when either summary has a result to act on, also while `summary.json` is older than a finished step, and gives each family `unreadable`; `state.json`'s family plugin adds `child` for a member; and `results.json` adds `saved_attention` for such a child, the flagged-card count of `summary.json`. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. @@ -266,7 +306,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The CYP2D6 depth check and the RFC1 motif count, ported to the Rust `og` binary: `og pgx cyp2d6-bed`, `og pgx cyp2d6-measure` and `og pgx cyp2d6-check` (from `bin/cyp2d6_depth_check.py`) and `og str motif` (from `bin/rfc1_motif.py`) read BAMs with htslib, without pysam. On 67 cases they write what the Python writes, byte for byte, with one fix: a mosdepth regions file that lists one region twice with two depths is now refused (exit 2); the Python averaged the two. Steps 21, 32 and 9b run them (see above); no option, output file or format changes. -- Genome map, first pieces, in the Rust `og` binary. `og map build` writes one person's map document: the chromosome bands, the runs of homozygosity, and every finding with a place on the genome (its own position, or its gene's span). A result the page keeps closed (a finding to act on, and APOE on every page) is tagged with its id on every row that could reveal it, so the page's own closed-result rule hides those rows too. `og map genes` builds the gene and exon files once per GENCODE release, offline. No pipeline output, option or page changes yet; the live page starts drawing the map in a later change. The format is in `rust/crates/og-map/README.md`. +- Genome map, first pieces, in the Rust `og` binary. `og map build` writes one person's map document: the chromosome bands, the runs of homozygosity, and every finding with a place on the genome (its own position, or its gene's span). A result the page keeps closed (a finding to act on, and APOE on every page) is tagged with its id on every row that could reveal it, so the page can keep APOE's rows behind its warning and mark a result opened on purpose. `og map genes` builds the gene and exon files once per GENCODE release, offline. No pipeline output, option or page changes yet; the live page starts drawing the map in a later change. The format is in `rust/crates/og-map/README.md`. - The run-log commands are now `og run state`, `og run eta`, `og run headroom` and `og run delta`, with the same options and the same output. The command lines of `bin/run_state.py`, `bin/run_eta.py`, `bin/run_headroom.py` and `bin/run_delta.py` are gone: run as commands they exit 1 and name the og command; the modules stay for the live site, its alerts, the database re-check and the summary, which import them. `scripts/generate-report.sh` writes the delta with `og run delta --newest-history `, so og picks the previous run by the one rule the summary uses too, and the report now needs og in `bin/.og/` (`scripts/setup.sh --og`), as steps 04, 05 and 07 do. On macOS, `bin/og` runs in a Linux container; docs/run-state.md says how to run the commands there. - `og run delta` copied every summary section once per section it compared and looked ClinVar hits up one by one. On a synthetic summary with 50,000 hits it took 24 s and 770 MB, against 0.3 s for the Python. It now borrows the sections and hashes the keys: 0.3 s and 310 MB, with the same three files byte for byte. diff --git a/CLAUDE.md b/CLAUDE.md index c713c6cfb..4b62adffb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,7 +138,7 @@ User's FASTQ/BAM/VCF 6. Add the module to the step list (a `plan` call) in `scripts/run-all.sh` (its `--tools` name and database parameter), so a default run includes it 7. Add a row for the image to `tests/smoke/commands.tsv`: a real command on the fixture and a check on what it wrote (an image with no row fails `container-test.yml`) 8. Update `docs/00-reference-setup.md` if new reference data needed -9. Update `docs/interpreting-results.md` if output needs explanation +9. Update the chapter page under `docs/results/` (listed in `docs/interpreting-results.md`) if output needs explanation 10. Test on at least one sample before committing ### Bumping a Tool @@ -185,7 +185,8 @@ Change its line in `versions.env`, plus the coupled data variable its comment na - **BWA-MEM2 index files**: Created alongside FASTA. Check for `.bwt.2bit.64`. - **ALIGN_DIR env var**: All alternative scripts accept `ALIGN_DIR=aligned_bwamem2`. - **TIDDIT --skip_assembly auto-detected**: Checks for BWA index files. -- **benchmark-variants.sh**: Pairwise mode (auto-discovers vcf*/ dirs) and truth set mode (hap.py). +- **benchmark-variants.sh**: Pairwise mode and truth set mode (hap.py). It compares `vcf/`, Strelka2's `vcf_strelka2/results/variants/variants.vcf.gz` and `.vcf.gz` in every `vcf_*/` dir (Clair3 included), the same glob `run-all.sh` uses for `BENCHMARK=true`. `comparison.tsv` has the allele-level Jaccard and `GT_Concordant`/`GT_Concordance`, the shared sites whose genotypes also agree. +- **03a runs without BQSR** and with `-ERC NONE`: not the full GATK Best Practices workflow, so its calls can differ from a clinical lab's GATK calls. ### bcftools - **`bcftools sort` requires `##contig` headers** — fails on VCFs without them. Inject from reference `.fai`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4929bcb3..1a8e266a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,7 @@ Every pipeline step needs: - `scripts/NN-tool-name.sh` — The executable script - An entry in the `nav` of [`zensical.toml`](zensical.toml), or the docs site check fails on the orphan page - A mention in the category table of [`docs/pipeline-overview.md`](docs/pipeline-overview.md) -- Section in `docs/interpreting-results.md` if the output needs explanation +- A section in the chapter page under `docs/results/` (listed in [`docs/interpreting-results.md`](docs/interpreting-results.md)) if the output needs explanation ### No Personal Data @@ -99,7 +99,7 @@ The CI pipeline enforces this with automated scanning. - `scripts/run-all.sh`: one entry in its step list (a `plan` call), so a default run passes the module's name in `--tools` (and its database parameter when the data is under `GENOME_DIR`). A bash-only step that a default run should include gets a switch there instead, run after the pipeline - [`tests/smoke/commands.tsv`](tests/smoke/commands.tsv): a row for the image, a real command on the fixture and a check on what the tool wrote. [`container-test.yml`](.github/workflows/container-test.yml) fails for an image with no row - `scripts/validate-setup.sh`: only if the step needs reference data to check - - `docs/interpreting-results.md`: add output interpretation + - the chapter page under `docs/results/` (listed in `docs/interpreting-results.md`): add output interpretation - `docs/00-reference-setup.md`: if new reference data is needed - `CLAUDE.md` — if the architecture tree changes 8. **Document failures** in `docs/lessons-learned.md` if you hit any issues during development. diff --git a/assets/imprinted_regions.tsv b/assets/imprinted_regions.tsv index c71bb24dd..b55146765 100644 --- a/assets/imprinted_regions.tsv +++ b/assets/imprinted_regions.tsv @@ -1,14 +1,17 @@ -# Imprinting disorders caused by uniparental disomy (UPD), one row per chromosome or region. -# Read by og summary collect (IMPRINTED_REGIONS in rust/crates/og-summary holds the same rows; its -# tests check that the two agree) to name what a run of homozygosity over most of one chromosome could mean. -# start and end: GRCh38, 0-based start, exclusive end, the UCSC cytoBand.hg38.txt bands of the region; -# empty for a row that applies to the whole chromosome. A row with a region applies when a run on the -# flagged chromosome overlaps it. "-" is no known imprinting disorder for UPD from that parent. -# Overview: Eggermann et al., Imprinting disorders, Nat Rev Dis Primers 2023 (PMID 37386011). +# Imprinted chromosomes and regions. Factual reference rows, curated by OwnGenome. +# Sole source: embedded by og summary collect at build time. The UPD hint emits region and PMID only; +# maternal_upd and paternal_upd remain reference facts here, never a diagnosis from a ROH match. +# start and end are retained for format compatibility and are empty: the hint is chromosome-wide. +# Mixed iso/heterodisomy may leave the imprinted locus outside the observed homozygous runs. +# Region labels are broad cytobands, not diagnostic locus coordinates or a specialist-reviewed assay. +# Overview: Eggermann et al., Imprinting disorders, Nat Rev Dis Primers 9, 33 (2023). +# https://doi.org/10.1038/s41572-023-00443-4 (PMID 37386011). +# Mechanism: Robinson, BioEssays 22:452-459 (2000), https://pubmed.ncbi.nlm.nih.gov/10797485/. +# Per-row PMIDs support the parent-of-origin reference facts; names and symbols are factual labels. chrom start end region maternal_upd paternal_upd pmid chr6 6q24 (PLAGL1, HYMAI) - Transient neonatal diabetes mellitus (6q24-related) 20301706 chr7 7p12 (GRB10), 7q32 (MEST) Silver-Russell syndrome - 27585961 -chr11 0 22000000 11p15 (H19/IGF2, KCNQ1OT1) Silver-Russell syndrome Beckwith-Wiedemann syndrome 29377879;27585961 +chr11 11p15 (H19/IGF2, KCNQ1OT1) Silver-Russell syndrome Beckwith-Wiedemann syndrome 29377879;27585961 chr14 14q32.2 (DLK1-MEG3) Temple syndrome Kagami-Ogata syndrome 24891339;26377239 -chr15 20500000 33400000 15q11-q13 (SNRPN, UBE3A) Prader-Willi syndrome Angelman syndrome 22237428;27615419 +chr15 15q11-q13 (SNRPN, UBE3A) Prader-Willi syndrome Angelman syndrome 22237428;27615419 chr20 20q13.32 (GNAS) Mulchandani-Bhoj-Conlin syndrome (growth failure) Pseudohypoparathyroidism type 1B 26248010;29959430 diff --git a/assets/live/core.js b/assets/live/core.js index 1e2f225c1..ed73d8a2d 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -146,19 +146,501 @@ return isFinite(x) && isFinite(y) && y > x ? (y - x) / 3600e3 : null; } - // --- the veil: a child's results to act on stay closed until someone opens them on purpose --------- - // The closed box, the same on a child's page (person.js) and on the family page (family-page.js): what stays - // closed, and the button that opens it. - function veil(text, button, onOpen) { - var v = el('div', { 'class': 'veil' }); - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), text)); - var b = el('button', { type: 'button', 'class': 'btn' }, button || 'Show the result'); - b.addEventListener('click', onOpen); - add(v, b); - return v; - } window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours, veil: veil }; + hm: hm, dm: dm, hours: hours }; + // --- the report gate ---------------------------------------------------------------------------------- + // A child's report stays closed while any of the child's results to act on (findings of level "act") is + // unopened. The page is then a function of the opened results alone and of how many are closed: no view (person.js, + // finding.js, map.js, family-page.js, home.js, live.js, the plugins) receives the child's report, because every + // view reads through fetch and EventSource and this gate holds both. Adults and children with no result to act on + // pass through untouched: each file is fetched as often, in the same order and with the same options as without + // the gate, and the event stream opens when the view asks for it. + var originalFetch = window.fetch.bind(window); + var records = {}, current = null, ownsPage = false, eventSources = [], shared = {}, cleaned = false; + var CLOSED_TEXT = "This child's report stays closed until every result to act on in adult life is opened on purpose."; + var TRANSFER = 'og-report-opening', NONCE = 'og-report-opening:'; + var findingPath = /\/finding\//.test(location.pathname) ? location.pathname : null; + // The opening ticket of the navigation that replaced the last document (transfer below): read and removed at + // once, before any network load, and valid only for this navigation (its one-use nonce in window.name). + var ticket = null; + try { + var text = sessionStorage.getItem(TRANSFER), t = text ? JSON.parse(text) : null; + sessionStorage.removeItem(TRANSFER); + if (window.name.indexOf(NONCE) === 0) { + if (t && typeof t.nonce === 'string' && window.name === NONCE + t.nonce && t.path === location.pathname) { ticket = t; } + window.name = t && typeof t.name === 'string' ? t.name : ''; + } + } catch (e) { ticket = null; } + function absolute(url, base) { return new URL(url, base || location.href).href; } + function copy(x) { return x === undefined ? null : JSON.parse(JSON.stringify(x)); } + function read(r) { + if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(location.pathname); } + return r.ok ? r.json() : null; + } + // The options the views read their JSON with (PGPUI.getJSON); live.js reads a replay's frames.json with none. + var NO_STORE = { credentials: 'same-origin', cache: 'no-store' }; + function jsonAt(url) { + return originalFetch(url, NO_STORE).then(read).catch(function () { return null; }); + } + // A file the gate reads before the views, with the options the view that reads it uses: its answer is kept once, + // whatever its status (a 404 or a failed request too), so the view's own first read of it costs no request. The + // view gets the response itself, and the kept answers reach the views in the order they arrived (inTurn), as the + // network would have handed them without the gate. + var arrivals = 0, claimed = []; + function keep(url, options) { + url = absolute(url); + var entry = shared[url] = { at: null }; + entry.kept = originalFetch(url, options).then(function (r) { + entry.response = r.ok ? r.clone() : r; + return r.ok ? r.text().then(function (t) { entry.text = t; }) : null; + }).catch(function () { entry.failed = true; }).then(function () { entry.at = ++arrivals; return entry; }); + return entry; + } + // What the gate itself reads of a kept answer: its JSON when it is ok, else null (a 401 goes to the login). + function parsedOf(entry) { + return entry.kept.then(function (e) { + if (e.failed) { return null; } + if (!e.response.ok) { return read({ status: e.response.status, ok: false }); } + return JSON.parse(e.text); + }).catch(function () { return null; }); + } + function sharedJSON(url, options) { return parsedOf(keep(url, options)); } + // A kept answer reaches its view in a task of its own, after every kept answer the views asked for that arrived + // earlier. Its json() gives the body the gate read, at once, so no answer overtakes another while it is parsed. + function inTurn(entry) { + claimed.push(entry); + return entry.kept.then(function () { + return new Promise(function (go, fail) { + (function wait() { + setTimeout(function () { + if (claimed.some(function (c) { return !c.handed && c.at !== null && c.at < entry.at; })) { wait(); return; } + entry.handed = true; + if (entry.failed) { fail(new TypeError('Failed to fetch')); return; } + var r = entry.response, text = entry.text; + if (r.ok) { r.json = function () { return Promise.resolve().then(function () { return JSON.parse(text); }); }; } + go(r); + }, 0); + }()); + }); + }); + } + function random() { var b = new Uint32Array(1); crypto.getRandomValues(b); return b[0] / 4294967296; } + // The results to act on, one record each: the findings of level "act" (bin/live_results.py). null when unreadable. + function actsOf(r) { + if (!r || !Array.isArray(r.findings)) { return null; } + var acts = [], seen = {}; + r.findings.forEach(function (f) { + if (f && f.level === 'act' && typeof f.id === 'string' && !seen[f.id]) { seen[f.id] = true; acts.push(f); } + }); + return acts; + } + function byId(rec, id) { return (rec.acts || []).filter(function (f) { return f.id === id; })[0]; } + function same(a, b) { return !!a && !!b && JSON.stringify(a) === JSON.stringify(b); } + // perm: the acts in a random order drawn for this page load. A closed control is named by its place in it + // (closed-1, closed-2), so no control's place says anything about the act behind it. order: the opened acts in + // the order they were opened (opened-1, opened-2). + function shuffled(ids) { + var out = ids.slice(); + for (var i = out.length - 1; i > 0; i--) { var j = Math.floor(random() * (i + 1)), x = out[i]; out[i] = out[j]; out[j] = x; } + return out; + } + function makeRecord(url, child, r, old) { + var acts = actsOf(r), rec = { url: url, child: child, acts: acts, opened: {}, order: [], perm: [], + ids: r && Array.isArray(r.findings) ? r.findings.map(function (f) { return f && f.id; }) : [], + attention: r && typeof r.attention === 'number' ? r.attention : null, + saved: r && typeof r.saved_attention === 'number' ? r.saved_attention : null }; + if (!acts) { return rec; } + // An act whose record changed is closed again; an unchanged one keeps its opening for this page. + rec.order = old ? old.order.filter(function (id) { return same(byId(old, id), byId(rec, id)); }) : []; + rec.order.forEach(function (id) { rec.opened[id] = true; }); + var ids = acts.map(function (f) { return f.id; }); + rec.perm = old && same(ids, (old.acts || []).map(function (f) { return f.id; })) ? old.perm : shuffled(ids); + return rec; + } + // PRE: the kept answer of results.json the gate asked for already (preparePerson), else it is read here. + function record(url, child, cached, pre) { + url = absolute(url); + return (pre ? parsedOf(pre) : cached ? sharedJSON(url + 'results.json', NO_STORE) : jsonAt(url + 'results.json')).then(function (r) { + var old = records[url]; + var rec = makeRecord(url, child !== undefined ? child : old ? old.child : true, r, old); + records[url] = rec; + return rec; + }); + } + function shut(rec) { return rec.child && (!rec.acts || rec.acts.some(function (f) { return !rec.opened[f.id]; })); } + function projection(rec) { + var closed = rec.perm.filter(function (id) { return !rec.opened[id]; }); + return { closed: closed.map(function (id, k) { return { handle: 'closed-' + (k + 1), id: id }; }), + opened: rec.order.map(function (id, k) { return { handle: 'opened-' + (k + 1), id: id, finding: copy(byId(rec, id)) }; }) }; + } + function handles(rec) { var p = projection(rec); return p.closed.concat(p.opened); } + // The address of a closed page names a handle or nothing: any other #f/ (an id, a guess) and any query + // land on the first closed handle, so a typed address tells nothing about which result is closed. + function normalizeHash(rec) { + if (!location.hash && !location.search) { return; } + var all = handles(rec), h = location.hash.slice(3); + var found = /^#f\/(closed|opened)-[1-9][0-9]*$/.test(location.hash) && all.some(function (a) { return a.handle === h; }); + var to = found ? location.hash : all.length ? '#f/' + all[0].handle : ''; + if (location.hash !== to || location.search) { history.replaceState(null, '', location.pathname + to); } + } + function actRecord(f) { + var box = el('div', { 'class': 'opened-act' }); + add(box, el('h2', {}, f.title), el('p', {}, f.detail), el('p', {}, f.text)); + var plan = el('ol', {}); + (f.plan || []).forEach(function (t) { add(plan, el('li', {}, t)); }); + add(box, plan, facts(f.evidence), caveat(f.caveat)); + return box; + } + // The controls of a closed report: the fixed text, the count, one block of identical closed controls, then each + // opened act's own record, in the order it was opened. PREFIX keeps element ids apart on a page with two children. + function controls(rec, changed, prefix) { + var p = projection(rec), box = el('section', { 'class': 'report-gate' }), n = (rec.acts || []).length; + add(box, el('p', { 'class': 'lede' }, CLOSED_TEXT)); + add(box, el('p', {}, rec.acts ? n + (n === 1 ? ' result' : ' results') + ' to act on' : 'The results cannot be read yet. Reload to try again.')); + function row(a, open) { + var art = el('article', { id: prefix + 'f-' + a.handle, 'data-finding': a.handle, 'class': 'finding' }); + if (open) { add(art, actRecord(a.finding)); } + var b = el('button', { type: 'button', 'class': 'btn', 'data-opening': a.handle }, open ? 'Close it again' : 'Show the result'); + b.addEventListener('click', function () { toggle(rec, a.id, open, changed, !prefix); }); + return add(art, b); + } + if (p.closed.length) { add(box, add.apply(null, [el('div', { 'class': 'closed-acts' })].concat(p.closed.map(function (a) { return row(a, false); })))); } + p.opened.forEach(function (a) { add(box, row(a, true)); }); + return box; + } + function toggle(rec, id, open, changed, addressed) { + var active = records[rec.url] || rec; + // A control drawn for an act record that has changed since opens nothing: the page draws again instead. + if (!same(byId(active, id), byId(rec, id))) { changed(); return; } + var at = addressed ? handles(active).filter(function (a) { return '#f/' + a.handle === location.hash; })[0] : null; + if (open) { delete active.opened[id]; active.order = active.order.filter(function (x) { return x !== id; }); } + else { active.opened[id] = true; active.order.push(id); } + if (at) { + // The address follows the act it named, under its new handle. + var now = handles(active).filter(function (a) { return a.id === at.id; })[0]; + history.replaceState(null, '', location.pathname + (now ? '#f/' + now.handle : '')); + } + changed(); + } + function cleanPage() { + // A closed page has no renderer, search widget, replay, canvas, progress view or external report link. + document.body.textContent = ''; + Array.prototype.slice.call(root.attributes).forEach(function (a) { + if (a.name.indexOf('data-pgp-') === 0 || a.name === 'data-finding') { root.removeAttribute(a.name); } + }); + document.title = 'Results closed: OwnGenome'; + } + function runPath() { return new URL('../../', absolute(findingPath)).pathname; } + function renderGate() { + cleanPage(); + // A finding's address names its finding: a closed page shows the run's address instead, the same for any id. + if (findingPath) { history.replaceState(null, '', runPath() + location.hash); } + normalizeHash(current); + var wrap = el('main', { 'class': 'wrap', id: 'report-gate' }); + add(wrap, el('h1', {}, 'Results about a child')); + add(wrap, controls(current, function () { if (shut(current)) { renderGate(); } else { transfer(); } }, '')); + add(document.body, wrap); + root.setAttribute('data-pgp-status', 'ready'); + root.setAttribute('data-pgp-gated', 'true'); + } + function proof(rec) { + var bytes = new TextEncoder().encode(JSON.stringify(rec.acts)); + return crypto.subtle.digest('SHA-256', bytes).then(function (h) { return Array.from(new Uint8Array(h)).map(function (v) { return v.toString(16).padStart(2, '0'); }).join(''); }); + } + // Replacing the document discards every view closure, image, chart, search cache and in-flight callback. Only + // which acts are open crosses it, with a digest of their records and a nonce for the next navigation only. + function transfer() { + proof(current).then(function (stamp) { + var ids = current.acts.map(function (f) { return f.id; }), b = new Uint8Array(16); + crypto.getRandomValues(b); + var nonce = Array.from(b).map(function (v) { return v.toString(16).padStart(2, '0'); }).join(''); + // A finding address releases to the finding's page only when the results have that finding; any other id + // goes to the run's page, as the server answers 404 for it once the report is open. + var id = null; + try { id = findingPath ? decodeURIComponent(findingPath.split('/finding/')[1].replace(/\/$/, '')) : null; } catch (e) { id = null; } + var target = !findingPath ? location.pathname : (current.ids || []).indexOf(id) >= 0 ? findingPath : runPath(); + sessionStorage.setItem(TRANSFER, JSON.stringify({ path: target, stamp: stamp, nonce: nonce, + name: window.name.indexOf(NONCE) === 0 ? '' : window.name, order: current.order.map(function (id) { return ids.indexOf(id); }) })); + window.name = NONCE + nonce; + location.replace(target); // a map or section address returns to the default view + }); + } + function consume(rec) { + var t = ticket; + ticket = null; + if (!t || !rec.acts || !Array.isArray(t.order)) { return Promise.resolve(); } + return proof(rec).then(function (stamp) { + if (t.stamp !== stamp) { return; } + t.order.forEach(function (i) { + var f = typeof i === 'number' ? rec.acts[i] : null; + if (f && !rec.opened[f.id]) { rec.opened[f.id] = true; rec.order.push(f.id); } + }); + }); + } + // The person and finding pages: the run's state and, for a child, its results, read in the order the views read + // them (the finding page asks for its results at once, the person page once the state has answered) and with the + // views' options. Who is a child is the server's one rule (bin/owngenome_live.py child_of), which state.json's + // family plugin carries (child, or unknown for a run no readable family lists while one cannot be read); a state + // that cannot be read cannot say the person is not a child. An adult's page then reads nothing more through the + // gate: its views ask for the family and the results themselves. For a child the gate reads the results first. Its + // family is asked for once the page's scripts have run, so live.js has opened its event stream first, and in live + // mode its results right after, as the person page asks for both on its first state: the two answers race as they + // do without the gate and reach the views in the order they arrived. + var parsed = new Promise(function (done) { + if (document.readyState === 'complete') { done(); } else { document.addEventListener('DOMContentLoaded', done); window.addEventListener('load', done); } + }); + function preparePerson() { + var url = absolute(findingPath ? '../../' : './'); + var replaying = root.getAttribute('data-mode') === 'replay'; + // live.js reads a replay's frames.json with no options; finding.js reads it, and state.json, as JSON (NO_STORE). + var stateRead = sharedJSON(url + (replaying ? 'frames.json' : 'state.json'), replaying && !findingPath ? undefined : NO_STORE); + var results = findingPath ? record(url, undefined, true) : null, pre = null; + return Promise.all([stateRead, parsed]).then(function (x) { + var raw = x[0]; + var state = replaying ? raw && raw.frames && raw.frames.slice(-1)[0] : raw; + state = replaying ? state && state.state : state; + var fam = state && state.plugins ? state.plugins.family : undefined; + var child = !state || !state.plugins || !!(fam && (fam.unknown || (fam.kind === 'member' && fam.child !== false))); + var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; + if (child && entry) { + keep(absolute(entry.url, url) + 'family.json', NO_STORE); + if (!replaying && !findingPath) { pre = keep(url + 'results.json', NO_STORE); } + } + return child; + }).then(function (child) { + if (!child && !results) { return { url: url, child: false, acts: [], opened: {}, order: [], perm: [] }; } + return (results || record(url, child, true, pre)).then(function (rec) { rec.child = child; return rec; }); + }).then(function (rec) { + current = records[url] = rec; + return consume(current); + }).then(function () { + ownsPage = shut(current); + if (ownsPage) { renderGate(); } + }).catch(function () { + current = { child: true, acts: null, opened: {}, order: [], perm: [] }; ownsPage = true; renderGate(); + }); + } + // A family's children with a run here, by the one rule (childIn, bin/owngenome_live.py child_of): family.json + // marks each member run `child` as the server decides it over every family, so a person who is a child in another + // family is one here too. A member who was never sequenced, or has no run, has no report to close. + function childRun(f, s) { var r = f.runs && f.runs[s]; return !!(r && r.url && r.child !== false); } + function familyRecords(f, base) { + return Promise.all(((f.state && f.state.members) || []).filter(function (m) { return childRun(f, m.sample); }).map(function (m) { + return record(absolute(f.runs[m.sample].url, base), true).then(function (r) { return [m.sample, r]; }); + })).then(function (entries) { var out = {}; entries.forEach(function (e) { out[e[0]] = e[1]; }); return out; }); + } + // A run's progress on the home and family pages: how far the run is, never what it found. + var PROGRESS = ['name', 'kind', 'url', 'state', 'started_utc', 'last_heard_utc', 'finished_utc', 'planned', 'done', 'running', 'failed']; + function progressOf(r) { var out = {}; PROGRESS.forEach(function (k) { if (r[k] !== undefined) { out[k] = r[k]; } }); return out; } + // A released child's flagged-card count: family.json leaves it out (null) while the server keeps the child's report + // closed, so once every result to act on is opened the page takes the count family.json gives without the gate, + // that of the child's summary.json, which the child's results.json carries as saved_attention. + function released(raw, recs) { + Object.keys(recs).forEach(function (k) { + var r = raw.runs && raw.runs[k]; + if (r && r.attention === null && !shut(recs[k])) { r.attention = recs[k].saved; } + }); + return raw; + } + function projectFamily(raw, recs) { + var closed = Object.keys(recs).filter(function (k) { return shut(recs[k]); }); + if (!closed.length) { return released(raw, recs); } + raw = released(raw, recs); + function gated(s) { return closed.indexOf(s) >= 0; } + // An ownership allowlist, not a classifier of genes, prose or individual report fields: a field a producer adds + // tomorrow stays out until it is listed here. + var f = { state: { kind: raw.state.kind, name: raw.state.name, pedigree: raw.state.pedigree, + members: (raw.state.members || []).map(function (m) { return { sample: m.sample, sex: m.sex, father: m.father, mother: m.mother, partner: m.partner, sequenced: m.sequenced }; }), + children: [], couples: [], plan: [], reports: [] }, + layout: raw.layout ? { width: raw.layout.width, height: raw.layout.height, size: raw.layout.size, lines: copy(raw.layout.lines), + people: (raw.layout.people || []).map(function (m) { return { sample: m.sample, sex: m.sex, sequenced: m.sequenced, x: m.x, y: m.y }; }) } : null, + couple_note: raw.couple_note, chrom_lengths: copy(raw.chrom_lengths), gated: closed, runs: {} }; + // Each person keeps the role this family gives them: its children are the ones it lists with a father or a + // mother. A gated person it lists as a parent or partner (a child in another family) is in gated alone, and the + // view gives them their buttons beside the children. + f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { + return gated(m.sample) ? { child: m.sample, gated: true } : copy((raw.state.children || []).filter(function (c) { return c.child === m.sample; })[0]); + }).filter(Boolean); + // A row belongs to a gated child when it names that child as one of its members, by exact name. + function names(v) { return String(v || '').split(/[\s,;]+/); } + // A couple with a gated partner keeps its names and whether it was screened; its screen is that partner's data. + f.state.couples = (raw.state.couples || []).map(function (c) { + if (!gated(c.a) && !gated(c.b)) { return copy(c); } + return c.screened ? { a: c.a, b: c.b, closed: true } : { a: c.a, b: c.b, report: null, table: null, screened: false, shared: 0, x_linked: 0 }; + }); + // The parents' own items stay: the plan rows that name no gated child, and the couple reports of couples with + // no gated partner. Any other report of the family folder may hold a child's data and stays out. + f.state.plan = copy((raw.state.plan || []).filter(function (r) { return !names(r.members).some(gated); })); + var reports = f.state.couples.map(function (c) { return c.closed ? null : c.report; }).filter(Boolean); + f.state.reports = copy((raw.state.reports || []).filter(function (r) { return reports.indexOf(r.url) >= 0; })); + function free(r) { return !gated(r.sample_a) && !gated(r.sample_b) && !gated(r.a) && !gated(r.b); } + f.relationships = copy((raw.relationships || []).filter(free)); + f.ibd_pairs = copy((raw.ibd_pairs || []).filter(free)); + f.couples = {}; + Object.keys(raw.couples || {}).forEach(function (key) { if (!key.split(' ').some(gated)) { f.couples[key] = copy(raw.couples[key]); } }); + ['segments', 'trio_status', 'trio_files', 'trio_file_state'].forEach(function (key) { + if (!raw[key] || typeof raw[key] !== 'object') { return; } + f[key] = {}; + Object.keys(raw[key]).forEach(function (k) { if (!gated(k)) { f[key][k] = copy(raw[key][k]); } }); + }); + ['de_novo', 'compound_het'].forEach(function (key) { f[key] = copy((raw[key] || []).filter(function (r) { return !gated(r.child); })); }); + Object.keys(raw.runs || {}).forEach(function (k) { + f.runs[k] = gated(k) ? Object.assign(progressOf(raw.runs[k]), { url: raw.runs[k].url, gated: true }) : copy(raw.runs[k]); + }); + f.openings = {}; closed.forEach(function (k) { f.openings[k] = (recs[k].acts || []).length; }); + if (raw.side_by_side) { + var sb = { members: copy(raw.side_by_side.members), rows: [] }, seenRows = Object.create(null); + // A row is there when an open member has a value in it, never because a gated child has one; the gated + // child's cell reads "closed". Rows come in a fixed order, so neither a row nor its place depends on the child. + (raw.side_by_side.rows || []).forEach(function (row) { + if (!row.values.some(function (v, i) { return !gated(sb.members[i]) && v !== null && v !== undefined && typeof v !== 'object' && ['', '.', '—', 'no summary yet', 'summary not readable'].indexOf(v) < 0; })) { return; } + var out = { group: row.group, label: row.label, + values: row.values.map(function (v, i) { return gated(sb.members[i]) ? 'closed' : v; }), + stale: (row.stale || []).map(function (v, i) { return gated(sb.members[i]) ? false : v; }) }; + var key = JSON.stringify(out); + if (!seenRows[key]) { seenRows[key] = true; sb.rows.push(out); } + }); + sb.rows.sort(function (a, b) { var x = JSON.stringify([a.group, a.label]), y = JSON.stringify([b.group, b.label]); return x < y ? -1 : x > y ? 1 : 0; }); + f.side_by_side = sb; + } + return f; + } + // Who is a child, the server's one rule (bin/owngenome_live.py child_of) over the families of runs.json: a run any + // readable family lists with a father or a mother; while a family cannot be read, every run while its members + // cannot be read (an empty member list) and the runs it lists while another of its files cannot be read. + function childIn(fams, name) { + function listed(f) { return (f.members || []).filter(function (m) { return m.sample === name; }); } + if (fams.some(function (f) { return listed(f).some(function (m) { return m.father || m.mother; }); })) { return true; } + return fams.some(function (f) { return !(f.members || []).length || (f.unreadable && listed(f).length > 0); }); + } + // The home page decides from runs.json alone: a child's summary is closed when runs.json counts or names a result + // to act on (the server counts a child's from the summary its results.json reads). + function projectRuns(data) { + var idx = copy(data), fams = idx.families || []; + idx.runs = (idx.runs || []).map(function (r) { + var n = (r.levels && r.levels.act) || (r.top || []).filter(function (x) { return x && x.level === 'act'; }).length; + if (!childIn(fams, r.name) || !n) { return r; } + return Object.assign(progressOf(r), { gated: true, results: true, screened: true, attention: null, levels: { act: n }, + top: [{ id: '', level: 'act', title: CLOSED_TEXT }], lead: CLOSED_TEXT, lead_closed: CLOSED_TEXT }); + }); + return idx; + } + var familyGates = {}, familyKids = []; + // The two files the gate projects before a view reads them: family.json on the family page and runs.json on the + // home page. Every other response reaches its view as it came. + var familyPage = document.body.classList.contains('page-family'); + function projected(path) { return familyPage ? path.endsWith('/family.json') : path.endsWith('/runs.json'); } + function loadedJSON(url, data) { + var u = new URL(url, location.href); + if (familyPage) { + familyKids = ((data.state && data.state.children) || []).map(function (c) { return c.child; }); + return familyRecords(data, u.href).then(function (recs) { familyGates = recs; return projectFamily(data, recs); }); + } + return Promise.resolve(projectRuns(data)); + } + var ready = Promise.resolve(); + if (document.body.classList.contains('page-person') || document.body.classList.contains('page-finding')) { ready = preparePerson(); } + // Permission is checked after parsing too: a response started while open may finish after closing. + function viewResponse(response) { + var parse = response.json.bind(response); + response.json = function () { return parse().then(function (data) { return ownsPage ? new Promise(function () {}) : data; }); }; + return response; + } + function closedRun(url) { return Object.keys(records).some(function (key) { return url.indexOf(key) === 0 && shut(records[key]); }); } + // Every view reads through here (and through EventSource below); one that reads a child's closed run gets nothing. + window.fetch = function (url, options) { + return ready.then(function () { + var requestURL = absolute(url && url.url || url); + if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); } + var kept = shared[requestURL]; + delete shared[requestURL]; + var got = kept ? inTurn(kept) : originalFetch(url, options); + return got.then(function (response) { + if (ownsPage) { return new Promise(function () {}); } + var path = new URL(requestURL).pathname; + if (response.ok && current && current.child && path === new URL(current.url + 'results.json').pathname) { + var check = function (r) { + var next = makeRecord(current.url, true, r, current); + if (!same(next.acts, current.acts)) { + current = records[current.url] = next; + // A result to act on that appeared or changed while the report was open closes it at once. + if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return new Promise(function () {}); } + } + return viewResponse(response); + }; + // A kept answer is checked on the text the gate read, in this task, so it reaches the view in its turn + // and no answer handed after it overtakes it. + var text = kept && kept.text; + if (typeof text === 'string') { + try { return check(JSON.parse(text)); } catch (e) { /* not JSON: read as the view would */ } + } + return response.clone().json().then(check); + } + if (response.ok && projected(path)) { + return response.json().then(function (data) { return loadedJSON(requestURL, data); }).then(function (data) { + return viewResponse(new Response(JSON.stringify(data), { status: response.status, headers: response.headers })); + }); + } + return viewResponse(response); + }); + }); + }; + window.addEventListener('hashchange', function () { if (ownsPage) { renderGate(); } }); + // A child's open report never stays in the back-forward cache: it is cleared on the way out, and a page shown + // again from that cache loads afresh (closed). Other pages come back from the cache as they were. + window.addEventListener('pagehide', function () { + if (current && current.child && current.acts && current.acts.length) { ownsPage = true; cleaned = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); } + }); + window.addEventListener('pageshow', function (e) { if (e.persisted && cleaned) { location.reload(); } }); + var NativeEvents = window.EventSource; + // A view's event stream opens the moment the view asks for it, as without the gate (live.js asks right after + // state.json). What it says waits until the gate knows whether it owns the page: then it reaches the view, in + // order, or the stream closes unread. After that it reaches the view only while the gate does not own the page. + window.EventSource = function (url, options) { + var proxy = this, held = [], closed = false; + var source = ownsPage || closedRun(absolute(url)) ? null : new NativeEvents(url, options); + Object.defineProperty(proxy, 'readyState', { get: function () { return closed || !source ? 2 : source.readyState; } }); + proxy.close = function () { closed = true; if (source) { source.close(); } }; + if (!source) { return; } + eventSources.push(source); + function deliver(kind, e) { if (!ownsPage && !closed && proxy['on' + kind]) { proxy['on' + kind](e); } } + ['message', 'error', 'open'].forEach(function (kind) { + source['on' + kind] = function (e) { if (held) { held.push([kind, e]); } else { deliver(kind, e); } }; + }); + ready.then(function () { + var q = held; + held = null; + if (ownsPage || closedRun(absolute(url))) { source.close(); return; } + q.forEach(function (x) { deliver(x[0], x[1]); }); + }); + }; + window.EventSource.CLOSED = NativeEvents.CLOSED; + window.PGPUI.ready = ready; + window.PGPUI.gateOwnsPage = function () { return ownsPage; }; + // Whether the gate treats this page's person as a child (the server's one rule), for person.js's closed page. + window.PGPUI.gateChild = function () { return !!(current && current.child); }; + window.PGPUI.closedText = CLOSED_TEXT; + // The acts the gate released, for person.js's map switches: every act of a child's open report, else none. + window.PGPUI.openedActs = function () { return current && !shut(current) ? copy(current.opened) : {}; }; + // Closing one act of a child's open report hands the page back to the gate, which closes it whole. + window.PGPUI.closeAct = function (id) { + if (!current || !current.child || !byId(current, id)) { return false; } + delete current.opened[id]; + current.order = current.order.filter(function (x) { return x !== id; }); + ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true; + }; + // What the family page knows of each child's results (for the tests): "DEMO-003:known", "DEMO-005:failed" for a + // child with no run here. + window.PGPUI.familyStatus = function () { + return familyKids.slice().sort().map(function (k) { return k + ':' + (familyGates[k] && familyGates[k].acts ? 'known' : 'failed'); }).join(' '); + }; + window.PGPUI.familyControls = function (child, redraw) { + function build() { + var box = controls(familyGates[child], function () { box.replaceWith(build()); redraw(); }, child + '-'); + box.setAttribute('data-gate-child', child); return box; + } + return build(); + }; }()); diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 575d1ee72..b02455086 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -6,9 +6,8 @@ // phone), each child's stretches painted by parent, the de novo and compound-het candidates, and the // members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. -// A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json -// gives the closed results (veil.spans), and while one is closed every place of this page that could carry one -// closes for that child, the same whichever result it is (the veil below). +// A child's report stays closed here as on the child's own page: the shared gate (core.js) reads each child's +// results.json and gives this view the family without that child's report data while a result to act on is unopened. (function () { 'use strict'; var U = window.PGPUI; @@ -53,8 +52,8 @@ var seq = (st.members || []).filter(function (m) { return m.sequenced; }).length; out.push([seq + ' genome' + (seq === 1 ? '' : 's') + ' of this family are read together here. ', false]); (st.couples || []).forEach(function (c) { + if (c.closed) { out.push([c.a + ' and ' + c.b + '’s carrier screen stays closed in chapter III. ', false]); return; } if (!c.screened) { out.push([c.a + ' and ' + c.b + ' are not screened as a couple yet. ', false]); return; } - if (coupleShut(f, c).length) { out.push([c.a + ' and ' + c.b + '’s carrier screen stays closed in chapter III. ', false]); return; } if (c.shared) { out.push([c.a + ' and ' + c.b + ' both carry a change in ' + c.shared + ' gene' + (c.shared > 1 ? 's' : '') + ': chapter III gives the chance per pregnancy for each. ', true]); } else { out.push([c.a + ' and ' + c.b + ' share no carrier gene in what the screen sees; ', false], ['it does not see everything. ', false]); } if (c.x_linked) { out.push([c.x_linked + ' X-linked finding' + (c.x_linked > 1 ? 's' : '') + ' bear on sons. ', true]); } @@ -74,94 +73,6 @@ } function memberUrl(f, s) { return f.runs && f.runs[s] ? f.runs[s].url : null; } - // --- the veil --------------------------------------------------------------------------------------- - // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child - // with no run here, or whose results.json has not answered or could not be read, cannot say whether a result to - // act on is closed (UNKNOWN). - // What the page shows of a child never depends on which result is closed or where it lies: while a child has any - // result to act on still closed here, or while the page cannot tell, every place that could carry one closes for - // that child in the same way, with one box each and the same words whatever the result. The places: the - // painting's grandparents, the map as an image, the de novo and compound-het rows, the child's medicines and HLA - // cells of the side by side, and the carrier screen of a couple the child is part of. A known result opens with - // its own click, from any of those boxes; the places open once the last one is. While the page cannot tell, each - // box opens only its own place. Opened results are kept for the life of this page, as on the child's page: a - // reload closes them again. They are kept per child and per run (two children can carry the same result id). - var UNKNOWN = '?'; - var GENE_GROUPS = { Medicines: true, HLA: true }; // the side-by-side groups whose rows name a gene - var kids = {}, opened = {}, last = null; - function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } - function isChild(f, s) { return (f.state.children || []).some(function (c) { return c.child === s; }); } - function loadKids(f) { - (f.state.children || []).forEach(function (c) { - var url = memberUrl(f, c.child); - var k = kids[c.child]; - if (!url) { kids[c.child] = { status: 'failed' }; return; } - if (k && k.url === url && k.busy) { return; } - kids[c.child] = k && k.url === url ? k : { status: 'pending', url: url }; - kids[c.child].busy = true; - U.getJSON(url + 'results.json').then(function (r) { - var cur = kids[c.child]; - if (!cur || cur.url !== url) { return; } - cur.busy = false; - var v = (r && r.veil) || {}; - // Results the server could not build come back without their findings: as unreadable as no answer. - var usable = !!(r && Array.isArray(r.findings)); - var acts = usable ? r.findings.filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }) : []; - // Spans that are not [id, ...] entries naming every result to act on cannot say which are closed: read - // as none given (UNKNOWN while there are results to act on). - var spansOk = Array.isArray(v.spans) && v.spans.every(function (s) { return Array.isArray(s) && typeof s[0] === 'string'; }) && - acts.every(function (id) { return v.spans.some(function (s) { return s[0] === id; }); }); - var next = usable ? { status: 'known', url: url, spans: spansOk ? v.spans : null, acts: acts } - : { status: 'failed', url: url }; - // An answer that cannot be read keeps what was known: the page never opens on a failed request. - if (!usable && cur.status === 'known') { return; } - var key = JSON.stringify([next.status, next.spans, next.acts]); - if (key === cur.key) { return; } - next.key = key; - kids[c.child] = next; - if (last) { redraw(); } - }).catch(function () { - var cur = kids[c.child]; - if (!cur || cur.url !== url) { return; } - cur.busy = false; - if (cur.status !== 'known' && cur.status !== 'failed') { cur.status = 'failed'; cur.key = 'failed'; if (last) { redraw(); } } - }); - }); - } - // The results of CHILD still closed, as the place BOX sees them: their ids, [UNKNOWN] while this page cannot read - // them (until BOX's own button opens BOX), or [] once every one is opened. Results with act findings but no - // veil.spans cannot say which they are: UNKNOWN too. - function shutOf(child, box) { - var k = kids[child]; - if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) { - return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN]; - } - var out = []; - (k.spans || []).forEach(function (s) { if (!opened[openKey(child, s[0])] && out.indexOf(s[0]) < 0) { out.push(s[0]); } }); - return out; - } - // The closed box of the place BOX of CHILD: WHAT says what stays closed; one button opens one result (BUTTON - // while the page cannot read the child's results, and then it opens this place only). - function veilBox(child, shut, box, what, button) { - if (!shut.length) { return null; } - var v = shut[0] === UNKNOWN - ? U.veil(what + ': closed, as this page cannot read ' + child + '’s results yet, so it cannot tell whether a ' + - 'result to act on in adult life is among them.', button, function () { opened[openKey(child, 'box:' + box)] = true; redraw(); }) - : U.veil(what + ': closed while ' + child + ' has a result to act on in adult life that is closed here.', null, - function () { opened[openKey(child, shut[0])] = true; redraw(); }); - v.setAttribute('data-veil-child', child); - v.setAttribute('data-veil-box', box); - return v; - } - - // A couple's carrier screen names the partners' carrier genes and their positions, so it closes while a partner - // who is a child of this family has a result still closed: [[that partner, their closed results]]. - function coupleShut(f, c) { - return [c.a, c.b].filter(function (m) { return isChild(f, m); }) - .map(function (m) { return [m, shutOf(m, 'couple:' + c.a + ' ' + c.b)]; }) - .filter(function (x) { return x[1].length; }); - } - // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. function pedigree(f) { var L = f.layout, st = f.state; @@ -228,7 +139,11 @@ var state = !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' : r.state === 'finished' ? 'Finished' : (r.state === 'stopped' ? 'Stopped, ' : '') + r.done + ' of ' + r.planned + ' steps'; add(txt, el('span', { 'class': 'person-state' }, state)); - if (r && r.results) { + // A person whose report is closed gets the fixed closed line, never a count; every other member the line + // it has without the gate. + if (r && r.gated) { + add(txt, el('span', { 'class': 'person-look' }, 'Report closed')); + } else if (r && r.results) { add(txt, el('span', { 'class': 'person-look' + (r.attention ? ' attn' : '') }, r.attention ? r.attention + ' worth a look' : 'Nothing flagged; see the caveats')); } @@ -237,6 +152,12 @@ add(ul, li); }); add(s, ul); + // A person whose report is closed and whom this family lists as a parent or partner (a child in another family) + // gets their buttons here, after the members: this family's chapter of children does not name them. + (f.gated || []).forEach(function (name) { + if (f.state.children.some(function (c) { return c.child === name; })) { return; } + add(s, el('h3', {}, name), U.familyControls(name, fetchFamily)); + }); return s; } @@ -259,19 +180,15 @@ if (!f.state.couples.length) { add(s, el('p', { 'class': 'dim' }, 'The pedigree names no partners.')); } f.state.couples.forEach(function (c) { var p = el('p', { 'data-couple': c.a + ' ' + c.b, 'class': 'couple-line' }); - if (!c.screened) { - p.textContent = c.a + ' and ' + c.b + ': not screened yet (step 39).'; + // A couple with a partner whose report is closed (core.js): the screen is that partner's data. + if (c.closed) { + p.textContent = c.a + ' and ' + c.b + ': the carrier screen stays closed while a partner’s report is closed.'; add(s, p); return; } - var cs = coupleShut(f, c); - if (cs.length) { - // The whole screen closes: its counts, its boxes, its table and its report. - p.textContent = c.a + ' and ' + c.b + ': the carrier screen stays closed here.'; + if (!c.screened) { + p.textContent = c.a + ' and ' + c.b + ': not screened yet (step 39).'; add(s, p); - cs.forEach(function (x) { - add(s, veilBox(x[0], x[1], 'couple:' + c.a + ' ' + c.b, 'The carrier screen of ' + c.a + ' and ' + c.b, 'Show the screen')); - }); return; } var rows = (f.couples || {})[c.a + ' ' + c.b] || []; @@ -389,17 +306,7 @@ // coloured by the grandparent it came from (or by the parent when no grandparent was sequenced). var SIDE = { paternal: 'father', maternal: 'mother' }; function painting(f, child, all) { - // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a - // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only the - // stretches over a result, or the painting only when one lies over it, would point at where the result lies, so - // while the child has any result still closed every stretch is drawn and listed from its parent alone, with no - // grandparent, basis or closed mark, behind one box. - var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.'; }); - var shut = anc ? shutOf(child, 'paint') : []; - var rows = !shut.length ? all : all.map(function (r) { - return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, - discordant: r.discordant, ancestor: '.', basis: 'parent' }; - }); + var rows = all; var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); var colours = Object.create(null), next = 1; @@ -444,8 +351,6 @@ }); add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); add(fig, leg); - // One box for the whole painting: each stretch shows its parent only. - add(fig, veilBox(child, shut, 'paint', 'The grandparents of ' + child + '’s stretches', 'Show the grandparents')); var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -468,6 +373,10 @@ return 'Not assessed for this child: ' + why + '.'; } f.state.children.forEach(function (c, i) { + if (c.gated) { + add(s, el('h3', {}, c.child), U.familyControls(c.child, fetchFamily)); + return; + } var d = el('details', { 'data-child': c.child }); if (i === 0) { d.open = true; } add(d, el('summary', {}, c.child + ': father ' + (c.father || 'not named') + ', mother ' + (c.mother || 'not named'))); @@ -476,14 +385,7 @@ if (rows && rows.length) { add(d, painting(f, c.child, rows)); } else if (c.segments) { - // The drawing as an image is coloured by grandparent too, and cannot close one stretch: it stays closed - // whole, as the painting does. - var shut = shutOf(c.child, 'map'); - if (shut.length) { - add(d, veilBox(c.child, shut, 'map', 'The chromosome map of ' + c.child + ' and its colours', 'Show the map')); - } else { - add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); - } + add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } else { add(d, el('p', { 'class': 'dim' }, 'No chromosome map yet: step 38 draws one for a child with a sequenced parent.')); } @@ -492,15 +394,9 @@ if (dn.length) { add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); - // Every row closes while a result is: a row closed on its own would say where the result lies. - var dshut = shutOf(c.child, 'denovo'); - if (dshut.length) { - add(d, veilBox(c.child, dshut, 'denovo', 'The de novo candidates of ' + c.child, 'Show the rows')); - } else { - add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], - rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, - r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); - } + add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], + rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, + r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } @@ -508,13 +404,8 @@ add(d, el('h3', {}, 'Compound-het pairs')); if (ch.length) { add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); - var hshut = shutOf(c.child, 'het'); - if (hshut.length) { - add(d, veilBox(c.child, hshut, 'het', 'The compound-het pairs of ' + c.child, 'Show the rows')); - } else { - add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], - rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); - } + add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], + rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } @@ -530,18 +421,11 @@ if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); - // A child's medicines and HLA cells name genes (a CPIC gene can be the gene of a result to act on, RYR1 or - // CACNA1S). Closing only the row of a closed result's gene would leave a gap in a list everyone knows, so - // while a child has any result still closed every medicines and HLA cell of the child's column reads "closed": - // every row stays, with the parents' values. The haplogroups and blood groups name no gene. - var shutBy = {}; - sb.members.forEach(function (m) { if (isChild(f, m)) { shutBy[m] = shutOf(m, 'side'); } }); // A value from a section older than its input (state "stale") says so in its own cell. - var group = null, rows = [], old = 0, closedCells = {}; + var group = null, rows = [], old = 0; sb.rows.forEach(function (r) { rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) { var m = sb.members[i]; - if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) { closedCells[m] = true; return 'closed'; } if (!(r.stale || [])[i]) { return v; } old++; return v + STALE; @@ -549,9 +433,6 @@ group = r.group; }); add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); - sb.members.forEach(function (m) { - if (closedCells[m]) { add(s, veilBox(m, shutBy[m], 'side', 'The medicines and HLA cells of ' + m, 'Show the cells')); } - }); if (old) { add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' + 'earlier run, and the latest run did not write it again; that member’s page says why.')); @@ -568,19 +449,13 @@ } if (f.state.reports.length) { var ul = el('ul', { 'class': 'lines' }); - // A couple report of a screen that stays closed is left out with it. - var shutReports = (f.state.couples || []).filter(function (c) { return c.report && coupleShut(f, c).length; }) - .map(function (c) { return c.report; }); - f.state.reports.forEach(function (r) { - if (shutReports.indexOf(r.url) < 0) { add(ul, add(el('li', {}), el('a', { href: r.url }, r.name))); } - }); + f.state.reports.forEach(function (r) { add(ul, add(el('li', {}), el('a', { href: r.url }, r.name))); }); add(s, ul); } return s; } function draw(f) { - last = f; var main = document.getElementById('family'); // A redraw (a result opened, a child's results answered) keeps each child's fold as the reader left it. var folds = {}; @@ -622,10 +497,9 @@ }); window.scrollTo(0, y); // What the page knows of each child's results (for the tests): "DEMO-003:known". - root.setAttribute('data-pgp-kids', Object.keys(kids).sort().map(function (c) { return c + ':' + kids[c].status; }).join(' ')); + root.setAttribute('data-pgp-kids', U.familyStatus()); root.setAttribute('data-pgp-status', 'ready'); } - function redraw() { if (last) { draw(last); } } // The page asks again every 30 s and redraws only when family.json changed (a member's run moved on, // step 39 finished). A failed first request says so, with a retry; a later one keeps what is shown. @@ -644,7 +518,6 @@ if (!f) { failed(); return; } var k = JSON.stringify(f); // Each child's results are asked again with the family: a result to act on can appear while a run goes. - if (!f.state.error) { loadKids(f); } if (k !== drawnKey) { drawnKey = k; draw(f); } }).catch(failed); } diff --git a/assets/live/finding.js b/assets/live/finding.js index 5f6fc3a31..f7a3c23f0 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -3,9 +3,11 @@ // the explanation, the plan under it, then the evidence (the numbers, the frequency on a log strip, the // reads with a link to the genome viewer), the caveat, the related findings and the family. Everything // comes from the run's results.json (bin/live_results.py findings); nothing is computed here about -// genetics. A result to act on about a child of the family stays closed until it is opened on purpose. +// genetics. A child's finding page draws only once the report gate (core.js) has released the child's report. (function () { 'use strict'; + window.PGPUI.ready.then(function () { + if (window.PGPUI.gateOwnsPage()) { return; } var U = window.PGPUI; if (!U) { return; } var el = U.el, add = U.add; @@ -72,30 +74,14 @@ root.setAttribute('data-pgp-status', 'ready'); return; } - // known, not family: the click that opens this finding draws again with family as it came, UNKNOWN kept. var unknown = family === UNKNOWN; var known = unknown ? null : family; - var me = known && (known.state.members || []).filter(function (m) { return m.sample === r.sample; })[0]; - // A family that does not list this person cannot say they are not a child: closed as well. - var child = unknown || (!!known && !me) || !!(me && (me.father || me.mother)); - // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: - // results.veil.findings, bin/live_results.py veil()) stays closed with it, as on the person page. - var tags = (r.veil && r.veil.findings) || {}; - function tagOf(x) { var t = tags[x.id]; return t && t !== x.id ? t : null; } - // Opening this finding opens it and the result it names, nothing else: another result keeps its own veil. - var opened = {}; - if (draw.open) { opened[f.id] = true; if (tagOf(f)) { opened[tagOf(f)] = true; } } - function shut(x) { return child && (x.level === 'act' ? !opened[x.id] : !!tagOf(x) && !opened[tagOf(x)]); } - // Closed: no title, gene or variant anywhere on the page (its title, crumb, cover, rail) until opened. - var closed = shut(f); - var title = !closed ? f.title : f.level === 'act' ? 'A result to act on' : 'A finding about a result to act on'; - document.title = title + ': ' + r.sample + ': OwnGenome'; - document.getElementById('finding-crumb').textContent = title; + document.title = f.title + ': ' + r.sample + ': OwnGenome'; + document.getElementById('finding-crumb').textContent = f.title; var sec = (r.journal || []).filter(function (s) { return s.n === f.section; })[0]; var same = (r.findings || []).filter(function (x) { return x.level === f.level; }); var order = (r.findings || []).filter(function (x) { return x.level !== 'info'; }); var next = order[order.indexOf(f) + 1]; - if (next && shut(next)) { next = null; } // The rail: back to the report, this page's parts, the next finding of the report. var rail = el('aside', { 'class': 'toc frail', 'aria-label': 'This finding' }); var back = el('a', { 'class': 'back', href: BASE + '#f/' + f.id }); @@ -124,14 +110,14 @@ add(lv, el('span', { 'class': 'dim' }, same.length === 1 ? 'The only finding at this level in ' + r.sample + '’s genome' : same.length + ' findings at this level in ' + r.sample + '’s genome')); add(cover, lv); - add(cover, el('h1', { 'class': 'h-display' }, title)); + add(cover, el('h1', { 'class': 'h-display' }, f.title)); var chips = el('div', { 'class': 'chips' }); var cs = []; if (f.gene) { cs.push(f.gene); } if (f.detail) { cs.push(f.detail); } if (f.viewer && cs.join(' ').indexOf(f.viewer) < 0) { cs.push(f.viewer); } cs.forEach(function (c) { add(chips, el('span', { 'class': 'chip mono' }, c)); }); - if (cs.length && !closed) { add(cover, chips); } + if (cs.length) { add(cover, chips); } var main0 = el('div', {}); add(main0, cover); var acts = el('aside', { 'class': 'actions' }); @@ -142,20 +128,6 @@ var top = el('div', { 'class': 'row' }); add(top, main0, acts); add(col, top); - if (closed) { - var v = el('div', { 'class': 'veil' }); - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), - (f.level === 'act' ? '' : 'This finding names a result to act on. ') + - 'A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.')); - var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { draw.open = true; draw(r, family); }); - add(v, b); - add(col, v); - add(body, rail, col); - add(main, body); - root.setAttribute('data-pgp-status', 'ready'); - return; - } var row1 = el('div', { 'class': 'row' }); var m1 = el('div', {}); add(m1, el('p', { 'class': 'lede' }, f.text)); @@ -197,7 +169,7 @@ } // Related findings and the family. var rel = (f.related || []).map(function (id) { return (r.findings || []).filter(function (x) { return x.id === id; })[0]; }) - .filter(function (x) { return x && !shut(x); }); + .filter(Boolean); if (known) { var sf = el('section', { 'class': 'sec', id: 'family' }); add(sf, add(el('div', { 'class': 'sechead' }), el('span', { 'class': 'secnum' }, ROMAN[parts.length - 3]), el('h2', { 'class': 'h2' }, 'In the family'))); @@ -247,7 +219,7 @@ var f = fam && fam.kind === 'member' && (fam.families || [])[0]; if (!f) { return Promise.resolve(null); } familyUrl = f.url; - // A family that does not answer: roles unknown, so a result to act on stays closed (draw()). + // A family that does not answer: roles unknown (the family part of the page is left out). return U.getJSON(abs(f.url) + 'family.json').then(function (d) { return d && d.state ? d : UNKNOWN; }).catch(function () { return UNKNOWN; }); } function failed() { @@ -266,4 +238,5 @@ }).catch(failed); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } + }); }()); diff --git a/assets/live/home.js b/assets/live/home.js index 6f25e7bb6..fed41b05c 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -192,7 +192,8 @@ }); items.sort(function (a, b) { return order[a.f.level] - order[b.f.level]; }); items.slice(0, 6).forEach(function (x) { - var a = el('a', { href: x.r.url + 'finding/' + encodeURIComponent(x.f.id) + '/' }); + // A child whose report the gate keeps closed (core.js) links to the run's page, never to a finding's own page. + var a = el('a', { href: x.r.url + (x.r.gated ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/') }); var who = el('span', { 'class': 'who' }); var pc = el('span', { 'class': 'pc' }, x.r.name); if (colourOf[x.r.name]) { pc.style.setProperty('--pcol', colourOf[x.r.name]); } diff --git a/assets/live/live.css b/assets/live/live.css index 6c0959048..f810abbde 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -202,10 +202,6 @@ section.sec { padding: 56px 0 8px; border-top: 1px solid var(--hair); margin-top .waitnote { border-left: 2px solid var(--brass-dim); padding: 4px 0 4px 14px; font-size: 13.5px; color: var(--ink-2); } .notassessed { border: 1px solid rgba(250, 178, 25, .35); border-radius: 8px; padding: 12px 14px; background: rgba(250, 178, 25, .05); font-size: 14px; color: var(--ink-2); margin: 0 0 12px; } .notassessed b { color: var(--rev); font-weight: 600; } -.veil { border: 1px dashed var(--hair-2); border-radius: 8px; padding: 14px 16px; display: flex; gap: 14px; align-items: center; margin: 14px 0; - background: repeating-linear-gradient(135deg, rgba(255, 255, 255, .015) 0 6px, transparent 6px 12px); } -.veil .vt { font-size: 13.5px; color: var(--ink-2); flex: 1; } -.veil .vt b { color: var(--ink); font-weight: 600; } /* section 8 and 9: assessed, no finding; what it cannot see */ .nf { list-style: none; padding: 0; margin: 0; } diff --git a/assets/live/map.js b/assets/live/map.js index 990d02b28..13cb03079 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -8,11 +8,13 @@ // stretch, the runs in view, how to move, and why a layer is not on the map. // // The veil. The document is the same for every reader: each row a closed result could reveal carries the id -// of that result (veil). Whether it is closed is the page's question (person.js veil()): a closed row is not -// drawn, has no hit target, no tooltip, no line in the margin and never reaches the address. The closed -// results are counted in a dashed chip that names no gene and no place, with an Open button that asks once, -// then opens the result through the page's own function, so the map and the card open together. A reload -// closes them again: the open state lives in the page's memory only. +// of that result (veil). Whether it is closed is the page's question (person.js mapVeil): a closed row is not +// drawn, has no hit target, no tooltip, no line in the margin and never reaches the address. A child's report +// never reaches this figure while one of its results to act on is unopened: the page's one gate (core.js) holds +// the whole page. So on a page the gate has released, a result to act on is opened, with a chip that closes it +// again through the page (which hands the page back to the gate), and the rows closed here are a gated result +// (APOE, behind its warning) until its own chip opens it, and a row whose result is not among the page's results. +// A reload closes them again: the open state lives in the page's memory only. // // No library and no build step: one canvas, the camera {ci, c, ls, k} (open chromosome, centre bp, natural // log of the span, 0 genome .. 1 open) eased toward its target each frame, one level() by bp per CSS pixel. @@ -132,8 +134,7 @@ // --- the figure --------------------------------------------------------------------------------------- // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), - // close(id), known(id) -> a finding of the page's results, closed() -> the ids of the results to act on the - // page keeps closed at this moment, stepDone(step) -> ended at this moment, fallback() -> rows}): one figure + // close(id), known(id) -> a finding of the page's results, stepDone(step) -> ended at this moment, fallback() -> rows}): one figure // per page. The page calls refresh(key) on each of its redraws; a new key // fetches the document again and keeps the camera. function create(opts) { @@ -207,8 +208,6 @@ var CW = 0, CH = 0, DPR = 1, phone = false; var cam = { ci: 0, c: 0, ls: 0, k: 0 }, tgt = { ci: 0, c: 0, ls: 0, k: 0 }; var hover = { row: -1, hit: null }, hits = [], raf = 0, settleT = 0, hashT = 0, confirming = null, touched = false; - // The results with no place on the map that its chip opened: each gets an "Opened in its section" chip while open. - var inSection = []; var cache = null, cacheKey = ''; // What the last frame drew one by one (mark() keys), the genome rows' share kept with their cache: the // browser check holds the In view table to it (canvas.pgpDrawn). @@ -862,16 +861,7 @@ function noFindingHere() { return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; } - function closedAny() { return closedActs().length > 0 || M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } - // The results to act on this page keeps closed, in the order Open opens them: the page's own order (the - // order of its closed cards), with a place on the map or without. Its length is what the chip counts, so - // neither the count nor the order tells whether a closed result has a place. A page that passes no list - // closes only the map's own rows, in map order. - function closedActs() { - var page = opts.closed ? opts.closed() : null; - if (page) { return page.filter(function (id) { return !M.gated[id]; }); } - return M.closable.filter(function (id) { return !M.gated[id] && stateOf(id) === 'closed'; }); - } + function closedAny() { return M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } function layerWords(lay) { if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } if (lay.state === 'unreadable') { return 'Not on the map: ' + (lay.source || '') + ' could not be read' + (lay.note ? ' (' + lay.note + ')' : '') + '.'; } @@ -1473,7 +1463,7 @@ function drawChips() { chips.textContent = ''; if (!M) { return; } - var acts = closedActs(), gates = [], opened = []; + var gates = [], opened = []; M.closable.forEach(function (id) { // A gated result (APOE) has a chip only once the page shows its card (a replay before its step has none). if (M.gated[id] && opts.now && !opts.now(id)) { return; } @@ -1482,17 +1472,11 @@ }); // Why it is closed: said beside the count, left out on a phone (the card says it too). function why() { return el('span', { 'class': 'gm-why' }, ' · Not drawn until opened on purpose'); } - if (acts.length) { - var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'act' }); - add(c, add(el('span', {}, acts.length === 1 ? '1 result closed' : acts.length + ' results closed'), why())); - add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); keyTo('.gm-yes'); })); - add(chips, c); - } gates.forEach(function (id) { var g = M.gated[id], name = g.gene || 'This result'; var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'gate' }); add(c, add(el('span', {}, name + ' closed'), why())); - add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); keyTo('.gm-yes'); })); + add(c, confirming === id ? confirmBox(id) : button('Open ' + name, function () { confirming = id; drawChips(); keyTo('.gm-yes'); })); add(chips, c); }); opened.forEach(function (id) { @@ -1502,27 +1486,17 @@ add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); add(chips, c); }); - // A result the chip opened that has no place: named only once it is open, with a link to its card. - inSection = inSection.filter(function (id) { return stateOf(id) !== 'closed' && opts.title && opts.title(id); }); - inSection.forEach(function (id) { - var c = el('div', { 'class': 'gm-chip opened', 'data-opened': 'section' }); - add(c, add(el('span', {}, 'Opened in its section: '), el('a', { href: '#f/' + id }, opts.title(id)))); - add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); - add(chips, c); - }); } // Where the keyboard goes once a chip's button rebuilt the chips: the chips' first button, else the map. function keyTo(sel) { var b = chips.querySelector(sel || 'button'); (b || canvas).focus({ preventScroll: true }); } - function confirmBox(id, gated) { + function confirmBox(id) { var box = el('span', { 'class': 'gm-confirm' }); - add(box, el('span', {}, gated ? 'It is kept behind its warning on every page. Open it here and in its section?' - : 'A result to act on in adult life is usually not looked at before the child can choose. Open it here and in its section?')); + add(box, el('span', {}, 'It is kept behind its warning on every page. Open it here and in its section?')); add(box, button('Show the result', function () { confirming = null; - if (!gated && M.closable.indexOf(id) < 0 && inSection.indexOf(id) < 0) { inSection.push(id); } opts.open(id); // the page draws again, and the map's chips with it (refresh) keyTo(); }, 'gm-yes')); diff --git a/assets/live/person.js b/assets/live/person.js index b4363b225..ba490119b 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -20,7 +20,8 @@ var results = null, state = null, family = null, familyFor = null, fetchedFor = null, drawnKey = null, folded = false; // 'none': in no family; 'pending' or 'failed': in one whose roles are not known (yet); 'known'. var familyStatus = 'none'; - var shown = {}; // veiled findings the reader opened, by id + var openingSeeded = false; + var shown = {}; // act openings transferred from the gate, plus the independent APOE switch var genomeMap = null, resultsKey = null; // Figure 1 (map.js), and the results it was built for var panel = document.getElementById('results'); var rail = document.getElementById('rail'); @@ -103,13 +104,17 @@ var ms = (family && family.data.state && family.data.state.members) || []; return ms.filter(function (m) { return m.sample === personName(); })[0] || null; } - // A child of the family: a member whose father or mother is listed. Results to act on about a child - // stay closed until someone opens them on purpose. - function isChild() { var m = me(); return !!(m && (m.father || m.mother)); } - // Results to act on stay closed for a child, and for anyone in a family whose roles are not known yet: - // the page cannot tell a child apart until family.json has answered. - // A family that answers without this person cannot say they are not a child either. - function closedPage() { return isChild() || familyStatus === 'pending' || familyStatus === 'failed' || (familyStatus === 'known' && !me()); } + // A page whose results to act on were opened on purpose: core.js releases a child's report, by the server's one rule + // (state.json's family plugin: a child in any family, everyone while a family's members cannot be read), only once + // each is opened, and the map marks them opened, with a chip to close them. The gate's answer alone decides: an + // adult's page is their normal page whatever family.json answers, or before it does. Only a page without the gate + // (the tests' reference, core.js with the gate cut out) keeps the rule it had before the gate, from family.json. + function closedPage() { + if (U.gateChild) { return !!U.gateChild(); } + var m = me(); + return !!(m && (m.father || m.mother)) || familyStatus === 'pending' || familyStatus === 'failed' || + (familyStatus === 'known' && !m); + } function memberUrl(s) { var r = family && family.data.runs && family.data.runs[s]; return r && r.url ? r.url : null; @@ -320,65 +325,27 @@ add(b, text); return b; } - // The findings of a card that a child's page keeps closed: while one is closed the card's own rows, - // numbers and drawing stay closed too, or its table would name the gene and variant under the veil. - // A row outside its result's card (a ClinVar row, a card's line, a viewer link, a chart's mark, a search hit) - // carries the id of the result to act on it names (results.veil, bin/live_results.py veil()): it stays closed - // while that result is, and "Show the result" opens it with it. A tag naming no finding of these results stays closed. - function tagClosed(id) { var f = findingById(id); return closedPage() && (!f || veiled(f)); } - function veilTags(kind, key) { return ((results && results.veil && results.veil[kind]) || {})[key] || []; } - // The charts (plugins/charts.js) ask for each mark. A closed page with no results.json yet cannot tell which - // marks those are, so it leaves every mark out until it can. + // This renderer receives a child's report only once the shared data gate (core.js) has released it, with every + // result to act on opened, so it closes no card row itself. The charts (plugins/charts.js) and the search box + // (plugins/search.js) still ask it, as before the gate, and on a page that keeps results closed until opened (a + // child's, by the gate's rule) get the same answers: before results.json has answered, + // every mark and hit is left out; then only a tag (results.veil) that names no finding of these results, since every + // result to act on here is open. Asked with no key, the search hears that the page keeps results closed, and says so. + function tagClosed(id) { return closedPage() && !findingById(id); } window.PGP.markClosed = function (chart, mark) { if (!closedPage()) { return false; } if (!results) { return true; } var id = ((results.veil && results.veil.marks && results.veil.marks[chart]) || {})[mark]; return !!(id && tagClosed(id)); }; - // A card's table without its closed rows: [table or null, how many rows it left out]. - function openTable(c) { - var tags = veilTags('rows', c.id), t = c.table; - if (!t || !t.rows || !tags.length) { return [t, 0]; } - var rows = t.rows.filter(function (r, i) { return !(tags[i] && tagClosed(tags[i])); }); - var out = {}; - Object.keys(t).forEach(function (k) { out[k] = t[k]; }); - out.rows = rows; - return [rows.length ? out : null, t.rows.length - rows.length]; - } - // A card's facts and drawing without their closed items (a fact, a CHIP call or a copy-number dot that names a - // closed result's gene): [facts, widget, how many items they left out]. - function openFacts(c) { - var ft = veilTags('facts', c.id), facts = (c.facts || []).filter(function (f, i) { return !(ft[i] && tagClosed(ft[i])); }); - var w = c.widget, key = w && (w.kind === 'vaf' ? 'calls' : w.kind === 'copies' ? 'genes' : null); - var wt = veilTags('widget', c.id), shut = (c.facts || []).length - facts.length; - if (key && wt.length && w[key]) { - var items = w[key].filter(function (x, i) { return !(wt[i] && tagClosed(wt[i])); }); - shut += w[key].length - items.length; - var copy = {}; - Object.keys(w).forEach(function (k) { copy[k] = w[k]; }); - copy[key] = items; - w = copy; - } - return [facts, w, shut]; - } - function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } - // The search box (plugins/search.js) asks the same question for each hit, by the summary key and the anchor - // of the section it comes from: a hit from a card this page keeps closed is left out. A closed page with - // no results.json yet cannot tell which cards those are, so it leaves every hit out until it can. - // Asked with no key, it answers whether this page keeps anything closed at all. - // A hit tagged with a result to act on (veil, bin/results_index.py) is left out while that result is closed. window.PGP.searchHidden = function (key, anchor, tag) { if (!closedPage()) { return false; } if (!results || key === null) { return true; } - if (tag && tagClosed(tag)) { return true; } - return cards().some(function (c) { - return (c.id === anchor || (c.keys || []).indexOf(key) >= 0) && veiledOf(c).length > 0; - }); + return !!(tag && tagClosed(tag)); }; function card(c) { var g = gate(c); - var closedFs = g === 'show' ? veiledOf(c) : []; var art = el('article', { 'class': 'card result g-' + g + (attentionNow(c) ? ' attn' : ''), id: 'card-' + c.id, 'data-section': c.id, 'data-step': (c.steps || []).join(' '), 'data-state': c.state, 'data-level': c.level || '', tabindex: '-1', 'aria-labelledby': 'h-' + c.id }); @@ -414,34 +381,18 @@ 'Step ' + (c.steps || []).join(', ') + ' failed: the step timeline says why, and a rerun of the step fills this section.')); } else if (g === 'empty') { add(text, el('p', { 'class': 'notassessed' }, 'Not assessed: its step ended without a result for this section.')); - } else if (closedFs.length) { - // One button opens one result: a card with two results to act on asks twice. - add(text, U.veil((closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + - ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, - function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } - var of = openFacts(c); - add(margin, U.facts(of[0])); - add(text, widget(of[1])); + add(margin, U.facts(c.facts)); + add(text, widget(c.widget)); Object.keys((results && results.charts) || {}).forEach(function (chart) { if (results.charts[chart] === c.id) { add(text, el('div', { 'class': 'chart-slot', 'data-chart-slot': chart })); } }); if (c.image) { add(text, el('img', { 'class': 'plot', src: c.image.src, alt: c.image.alt })); } - var ot = openTable(c), tb = U.table(ot[0]); - var lt = veilTags('lines', c.id); - var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); - var shut = ot[1] + (c.lines || []).length - lines.length, items = of[2]; - if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } - if (shut || items) { - // Rows of its table and lines, and items of its facts and drawing, counted apart. - var what = [shut ? (shut === 1 ? 'One row' : shut + ' rows') : '', items ? (items === 1 ? (shut ? 'one item' : 'One item') : items + ' items') + - ' of its numbers or drawing' : ''].filter(Boolean).join(' and '); - var many = shut + items > 1; - add(text, el('p', { 'class': 'dim veil-rows' }, what + (many ? ' stay' : ' stays') + - ' closed with the result to act on ' + (many ? 'they belong' : 'it belongs') + ' to, until that result is opened.')); - } + var tb = U.table(c.table); + var lines = c.lines || []; + if (tb) { add(text, tb); } else if (c.empty_text) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } add(text, lineList(lines)); if (c.hidden) { // APOE and the like: behind their warning, opened on purpose. @@ -463,8 +414,7 @@ add(margin, U.caveat(c.caveat)); var foot = el('p', { 'class': 'card-links' }); add(foot, el('a', { href: 'report.html#' + encodeURIComponent(c.anchor), 'class': 'report' }, 'Open in the report')); - var vtags = veilTags('viewer', c.id); - var reads = (c.viewer || []).filter(function (x, i) { return !(vtags[i] && tagClosed(vtags[i])); }); + var reads = c.viewer || []; if (g === 'show' && reads.length) { add(foot, el('a', { href: 'viewer/#' + reads[0], 'class': 'viewer' }, 'See the reads at ' + reads[0])); } @@ -536,61 +486,40 @@ add(box, d); return note('Caveat', box, 'caveat-note'); } - // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: results.veil.findings) - // stays closed while that result is, and opening it opens that result. - function findingTag(f) { var t = ((results && results.veil && results.veil.findings) || {})[f.id]; return t && t !== f.id ? t : null; } - function veiled(f) { - if (!closedPage()) { return false; } - if (f.level === 'act') { return !shown[f.id]; } - var t = findingTag(f); - return !!(t && tagClosed(t)); - } - function openFinding(f) { shown[f.id] = true; var t = findingTag(f); if (t) { shown[t] = true; } } - // The genome map's veil (map.js): 'closed' while this page keeps the result closed, 'opened' once it was - // opened on purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is - // closed on every page, an adult's too, until opened; a tag naming no finding of these results stays closed. + // The genome map's veil (map.js): 'opened' for a result to act on this page shows because it was opened on + // purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is closed on every + // page, an adult's too, until opened; a tag naming no finding of these results stays closed. function mapVeil(id, gated) { if (gated) { return shown[id] ? 'opened' : 'closed'; } if (!closedPage()) { return 'open'; } var f = findingById(id); - if (!f || veiled(f)) { return 'closed'; } - return f.level === 'act' || findingTag(f) ? 'opened' : 'open'; + if (!f) { return 'closed'; } + return f.level === 'act' ? 'opened' : 'open'; } - // One switch for a result, from the map's chip or the card's "Show the result" (openFinding): opening it in - // one place opens it in both. Closing it closes the findings that were opened with it. + // One switch for a result, from the map's chip or the APOE fold: opening it in one place opens it in both. + // Closing a child's result to act on hands the page back to the gate (core.js closeAct), which closes it again. function setShown(id, on) { - var f = findingById(id); - if (on) { if (f) { openFinding(f); } else { shown[id] = true; } } else { - delete shown[id]; - ((results && results.findings) || []).forEach(function (x) { if (findingTag(x) === id) { delete shown[x.id]; } }); - } + if (!on && U.closeAct(id)) { return; } + shown[id] = !!on; redraw(true); } // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws // the run's end results from its first frame). A row with no finding of these results (the map's own APOE // row) follows the card that holds its fold (the traits card): a replay before that card shows has no APOE. - // A finding this page keeps closed is not shown either, whatever row tag it shares with another result. function mapNow(id) { var f = findingById(id); if (!f) { var t = cards().filter(function (c) { return c.hidden; })[0]; return !!(t && gate(t) === 'show'); } var c = cardById(f.card); - return !!(c && gate(c) === 'show') && !veiled(f); - } - // The results to act on this page keeps closed at this moment, in the page's order: the map's chip counts - // them, whether or not they have a place on the map. - function mapClosed() { - return findingsNow().filter(function (f) { return f.level === 'act' && veiled(f); }).map(function (f) { return f.id; }); + return !!(c && gate(c) === 'show'); } - // The title of a result once the page shows it, for the map's "Opened in its section" chip; never a closed one's. - function mapTitle(id) { var f = findingById(id); return f && !veiled(f) ? f.title : ''; } // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). function mapKnown(id) { return !!findingById(id); } // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. function stepDone(step) { var s = stepsBy()[step]; return !s || s.state === 'done'; } // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings - // with a place, as a table, each through this page's veil first. + // with a place, as a table. function mapFallback() { - return findingsNow().filter(function (f) { return f.viewer && !veiled(f); }).map(function (f) { + return findingsNow().filter(function (f) { return f.viewer; }).map(function (f) { return [U.LEVEL_WORDS[f.level] || f.level, f.title, f.viewer]; }); } @@ -600,12 +529,6 @@ var at = newAt(f); if (at) { art.classList.add('new'); add(art, el('span', { 'class': 'newband' }, 'New at ' + U.hm(at))); add(art, el('br', {})); } add(art, U.findingLevel(f.level, f.level_words)); - if (veiled(f)) { - add(art, U.veil('A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.', - null, function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); - add(row, art, el('aside', {})); - return row; - } var h = el('h3', { 'class': big ? 'h1' : 'h3' }); add(h, el('a', { href: findingHref(f) }, f.title)); add(art, h); @@ -622,7 +545,7 @@ var ev = evidence(f.evidence); if (ev) { add(side, note('Evidence', ev)); } add(side, caveatNote(f.caveat, f.id)); - var rel = (f.related || []).map(findingById).filter(function (r) { return r && !veiled(r); }); + var rel = (f.related || []).map(findingById).filter(function (r) { return r; }); if (rel.length) { var box = el('div', {}); rel.slice(0, 4).forEach(function (r, i) { if (i) { add(box, el('br', {})); } add(box, el('a', { href: '#f/' + r.id }, r.title)); }); @@ -633,13 +556,6 @@ } function findingLine(f) { var li = el('li', { id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); - if (veiled(f)) { - // A finding that names a closed result (a symptom ranking of its gene): closed with it, opened with it. - var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } }); - add(li, el('span', { 'class': 't' }, 'A result about a child, closed until opened.'), U.findingLevel(f.level, f.level_words), b); - return li; - } add(li, el('a', { 'class': 't', href: findingHref(f) }, f.title), U.findingLevel(f.level, f.level_words)); if (f.detail) { add(li, el('span', { 'class': 'gene' }, f.detail)); } add(li, el('span', { 'class': 'x' }, f.text)); @@ -765,8 +681,7 @@ // One line for a card: what it found, or why it has nothing to show yet. function cardLine(c) { var g = gate(c); - return g === 'show' && veiledOf(c).length ? 'A result about a child, closed until opened.' - : g === 'show' ? (c.headline || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) + return g === 'show' ? (c.headline || ((c.facts || [])[0] ? c.facts[0][0] + ': ' + c.facts[0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } // A card the report flags that has no finding of its own: on a phone it is the only place that flag @@ -840,15 +755,6 @@ n.info = cards().filter(function (c) { return gate(c) === 'show' && c.level === 'info'; }).length; return n; } - // A segment of the lead that names a closed result, without the names: the list after a colon, the genes in - // brackets, or what follows "of". - function closedLead(t) { - var w = 'closed on this page until opened on purpose, in section 1'; - if (/^\s*:/.test(t)) { return ': ' + w + '. '; } - if (/\(.*\)/.test(t)) { return t.replace(/\(.*\)/, '(' + w + ')'); } - if (/ of /.test(t)) { return t.replace(/ of .*$/, ' result, ' + w); } - return w + '. '; - } function drawLead() { var slot = document.getElementById('lead-slot'); slot.textContent = ''; @@ -856,12 +762,7 @@ var row = el('div', { 'class': 'row' }); var main = el('div', {}); var p = el('p', { 'class': 'lede dropcap', id: 'lead' }); - var closed = (results.findings || []).some(veiled); - var leadTags = (results.veil && results.veil.lead) || []; - var parts = (results.lead || []).map(function (x, i) { - if (x[2] === 'act' && closed) { return [': closed on this page until opened on purpose, in section 1. ', false]; } - return leadTags[i] && tagClosed(leadTags[i]) ? [closedLead(x[0]), x[1]] : x; - }); + var parts = results.lead || []; if (replay && running()) { // A replay holds the results of the finished run: before its end the lead would name findings the // sections do not show yet. @@ -879,8 +780,8 @@ var a = el('a', { href: '#s' + x[1], 'data-level': x[0] }); add(a, U.level(x[0], { act: 'Take to a doctor', medicine: 'Prescriber', review: 'Worth a look', carrier: 'Family planning', info: 'Information' }[x[0]])); add(a, el('span', { 'class': 'n' }, String(n[x[0]]))); - var words = x[0] === 'info' ? 'Origins, traits, scores' : x[0] === 'act' && closed && n.act ? 'Closed until opened' : - fs.filter(function (f) { return f.level === x[0] && !veiled(f); }).slice(0, 2).map(function (f) { return f.gene || f.title; }).join(', ') || + var words = x[0] === 'info' ? 'Origins, traits, scores' : + fs.filter(function (f) { return f.level === x[0]; }).slice(0, 2).map(function (f) { return f.gene || f.title; }).join(', ') || (fs.some(function (f) { return f.level === x[0]; }) ? 'Closed until opened' : 'None in what was assessed'); add(a, el('span', { 'class': 'w' }, words)); add(led, a); @@ -888,7 +789,7 @@ add(main, led); var trust = el('div', { 'class': 'trustline' }); var problems = fs.filter(function (f) { return f.level === 'trust'; }); - var openProblems = problems.filter(function (f) { return !veiled(f); }); + var openProblems = problems; add(trust, U.level('trust', 'Data quality', problems.length ? 'problem' : '')); var qc = cardById('qc'); if (qc && gate(qc) === 'show') { @@ -906,10 +807,10 @@ add(row, main, side); add(slot, row); // Figure 1: the genome map (map.js), one figure kept across redraws so its camera stays where the - // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. + // reader left it. It asks this page's veil for each row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, title: mapTitle, stepDone: stepDone, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); @@ -1217,6 +1118,7 @@ window.PGP.register({ name: 'person', onState: function (s) { + if (!openingSeeded) { shown = U.openedActs(); openingSeeded = true; } state = s; loadFamily(); if (!replay || !results) { load(); } diff --git a/assets/live/plugins/charts.js b/assets/live/plugins/charts.js index 6808a282d..28a896f3f 100644 --- a/assets/live/plugins/charts.js +++ b/assets/live/plugins/charts.js @@ -4,10 +4,11 @@ // a mark opens a panel with the mark's detail, the chart's caveat, a link to the reads in the // genome viewer (viewer/#) and a link to the section of the report. On a person's page each // chart goes into its card (a [data-chart-slot=] person.js draws), else into this plugin's -// panel. A mark the page keeps closed (a result to act on about a child, person.js's PGP.markClosed) is -// left out of the drawing, its title and its detail with it, wherever the figure sits; the figure is drawn -// again with it once the result is opened. No inline script, no style attribute, no eval: the SVG is -// parsed as XML and holds no script. +// panel. A mark the page keeps closed (person.js's PGP.markClosed: on a child's page, every mark until the +// results have answered; a child's report with a result to act on still closed never reaches the page, see +// the report gate in core.js) is left out of the drawing, its title and its detail with it, wherever the +// figure sits; the figure is drawn again with it once the page can show it. No inline script, no style +// attribute, no eval: the SVG is parsed as XML and holds no script. (function () { 'use strict'; if (!window.PGP) { return; } diff --git a/assets/live/plugins/search.js b/assets/live/plugins/search.js index 997907fff..5c7719d34 100644 --- a/assets/live/plugins/search.js +++ b/assets/live/plugins/search.js @@ -85,10 +85,11 @@ return li; } - // A hit from a section the page keeps closed (a result to act on about a child, person.js's - // PGP.searchHidden) is left out, and so is a suggested term that only such sections name: the search - // must not name what the page veils. The page says so whether or not anything was left out. - // A hit tagged with a result to act on (it.veil) is left out while that result is closed, whatever its section. + // A hit the page keeps closed (person.js's PGP.searchHidden: on a child's page, every hit until the results + // have answered, and a hit tagged (it.veil) with a result to act on that is not open) is left out, and so is a + // suggested term that only such hits name. A child's report with a result to act on still closed never reaches + // the page (the report gate in core.js). A page that keeps results closed says so whether or not anything was + // left out. function hidden(it) { return !!(root.PGP.searchHidden && root.PGP.searchHidden(it.section, it.anchor, it.veil || null)); } function closedNote() { return root.PGP.searchHidden && root.PGP.searchHidden(null, null) diff --git a/assets/monitoring/pgp-alerts.yml b/assets/monitoring/pgp-alerts.yml index 3b69b3c7e..365ada731 100644 --- a/assets/monitoring/pgp-alerts.yml +++ b/assets/monitoring/pgp-alerts.yml @@ -1,5 +1,7 @@ -# Example Prometheus alert rules for OwnGenome runs, on the metrics of bin/owngenome_live_ops.py -# (GET /metrics of the live site). docs/alerts.md says how to scrape them and what each one means. +# Example Prometheus alert rules for OwnGenome runs, on the metrics of the live site's ops module +# (GET /metrics: bin/owngenome_live_ops.py, and og-live's ops module, rust/crates/og-live/src/ops.rs, +# which writes the same text). docs/alerts.md says how to scrape them and what each one means. +# rust/crates/og-live/tests/ops.rs checks that every metric these rules use is one /metrics writes. # The thresholds the page also shows (disk warn percent, stall minutes) are flags of the site, # so the page and these rules always agree. Each disk is compared with the warn ratio of the site # it came from (on job and instance), so one Prometheus can scrape several sites; the 'for' times and the 98% disk line are yours to tune. diff --git a/bin/live_results.py b/bin/live_results.py index 3f0a3a603..975b8a6ff 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -22,7 +22,9 @@ Beside the cards, "veil" tags the rows that name a result to act on outside its own card (every card's table rows and lines, the viewer links, the charts' marks, the other findings and the lead's names) with -that result's id, so a child's page closes them with it (veil() below). +that result's id (veil() below). A child's page no longer closes them one by one: one gate in +assets/live/core.js holds the whole report while a result to act on is unopened, and of "veil" the page +still reads only the charts' marks. Nothing here reads a file or computes anything about genetics; the adapters pick fields the summary already holds. A section id with no adapter falls back to its @@ -793,8 +795,9 @@ class _Row: cannot read (a field of the wrong type, a stars value that is not a number) costs that row only: it becomes a finding on its card that says the row could not be read, and every other row and card still renders. Its words name no gene or variant: the row is unreadable. Its level is the highest the rule - can give (level "act" for a rule that can give one): a child's page veils a card that holds a finding - to act on, and the card's table may still draw the row the rule could not read.""" + can give (level "act" for a rule that can give one): a child's report stays closed while a finding to act + on is unopened (the gate in assets/live/core.js), and the card's table may still draw the row the rule could + not read.""" def __init__(self, out, card_id, level="review"): self.out, self.card_id, self.level = out, card_id, level @@ -1137,10 +1140,11 @@ def lead(found, levels, summary): # --- the veil's tags ----------------------------------------------------------------------------- -# A child's page keeps each result to act on closed (assets/live/person.js veiled()). The page closes a -# finding's own card with it, but other places can name the same gene or variant: any card's table and -# lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. Each such row carries the id of the -# result it belongs to, so the page closes it with that result and opens it with it. The rule: a row +# Places outside the own card of a result to act on can name the same gene or variant: any card's table and +# lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. +# Each such row carries the id of the result it belongs to. A child's page no longer closes them one by one: +# one gate (assets/live/core.js) holds the whole report while a result to act on is unopened, and of these +# tags the page still reads only the charts' marks (veil() below). The rule: a row # belongs to a closed result when it names the result's gene, or its position falls inside the result's # position or span; a row that is the result's own position or span belongs to that result first, else # the first such result in finding order. The veil lives in the browser: the files still hold every row. @@ -1204,16 +1208,18 @@ def veil_id(spans, text="", chrom=None, pos=None, end=None): def veil(summary, found, chapters, lead_parts=()): - """The rows a child's page closes with a result to act on, by the result's id: each card's table rows + """Which rows outside a result's own card name a result to act on, by the result's id: each card's table rows (rows: card -> one id or None per row; the ClinVar card's by each hit's own position), each card's viewer links (viewer: card -> one per link), each card's lines (lines: card -> one per line; a known deep intronic site is printed whether hit or not), the marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" - names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items - of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself: the family - page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child - that could carry one while one is. Empty when nothing can be closed.""" + names left out), each card's facts (facts: card -> one per fact) and the items of its drawing (widget: card + -> one per CHIP call or copy-number dot); spans is closable() itself. A child's page no longer closes these + rows one by one: its one gate (assets/live/core.js) holds the whole report while a result to act on is + unopened and reads the findings of level "act" for it. Of this field only marks is still read, by person.js, + for a chart mark on a page the gate released (the search index and the genome map carry their own tags); the + other subfields stay so that no field of results.json is removed. Empty when nothing can be closed.""" spans = closable(found) if not spans: return {} diff --git a/bin/owngenome_live.py b/bin/owngenome_live.py index a712e43a4..c970ac660 100755 --- a/bin/owngenome_live.py +++ b/bin/owngenome_live.py @@ -105,6 +105,23 @@ def valid_run(name): return bool(name) and name not in (".", "..") and bool(RUN_RE.match(name)) +def child_of(families, name): + """Whether NAME's pages treat it as a child, the one rule of the person page (state.json's plugins.family.child, + from bin/owngenome_live_family.py member_state), the home page (assets/live/core.js childIn), the family page + (family.json's child) and the finding route: a run that any readable family lists with a father or a mother is a + child, whatever the other families say. While a family cannot be read, a run it might list is treated as a child + too (fail closed): every run while its members cannot be read (an empty member list), and the members it lists + while another of its files cannot be read (unreadable). Computed from the files on each request. + families: [{"name", "members": [{"sample", "father", "mother"}], "unreadable"}], as /runs.json lists them.""" + fams = families or [] + + def listed(f): + return [m for m in f.get("members") or [] if m.get("sample") == name] + if any(m.get("father") or m.get("mother") for f in fams for m in listed(f)): + return True + return any(not (f.get("members") or []) or (f.get("unreadable") and listed(f)) for f in fams) + + def utc_now(): return datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -529,6 +546,7 @@ def __init__(self, app, cfg, password_hash): self.headline_cache = {} self.attention_cache = {} self.live_overview = {} # run -> (the headline cache key, live_results.overview of that summary) + self.summary_overviews = {} # run -> (the summary summary() gave, live_results.overview of it) self.allowed = set(LOOPBACK_HOSTS) | {h.lower() for h in cfg.allowed_host or []} if cfg.bind not in ("0.0.0.0", "::"): self.allowed.add(cfg.bind.strip("[]").lower()) @@ -817,6 +835,35 @@ def overview(self, d): self.attention_cache[p] = (stamp, n) return n, mtime + def child(self, name, runs=None): + """child_of() over the families the server lists now.""" + return child_of([r for r in (runs or self.runs()).values() if r.get("kind") == "family"], name) + + def summary_overview(self, name, d, st): + """live_results.overview of the summary results.json reads now (summary()), cached with that summary; + None while there is none.""" + s = self.summary(name, d, st) + if not s: + return None + hit = self.summary_overviews.get(name) + if not hit or hit[0] is not s: + try: + import live_results + hit = (s, live_results.overview(s)) + except Exception as e: # one bad summary must not take a page down + print(f"owngenome_live: overview of {name} not built: {type(e).__name__}", file=sys.stderr) + hit = (s, None) + self.summary_overviews[name] = hit + return hit[1] + + def gated(self, name, d, st, runs=None): + """Whether NAME's own page keeps its report closed (assets/live/core.js): a child whose results hold a + result to act on, counted from the summary its results.json reads (summary.json while that has none).""" + if not self.child(name, runs): + return False + o = self.summary_overview(name, d, st) or self.overview(d)[0] + return bool(((o or {}).get("levels") or {}).get("act")) + def runs_index(self): """/runs.json: each run's progress, and the families with their members, for the home page. No result beyond the attention count, the counts by level, the titles of the first four @@ -827,6 +874,8 @@ def runs_index(self): self.headline_cache.pop(k, None) for k in [k for k in list(self.live_overview) if k not in runs]: self.live_overview.pop(k, None) + for k in [k for k in list(self.summary_overviews) if k not in runs]: + self.summary_overviews.pop(k, None) keep = {os.path.join(r["dir"] or os.path.join(self.root, n), "summary.json") for n, r in runs.items()} for k in [k for k in list(self.attention_cache) if k not in keep]: self.attention_cache.pop(k, None) @@ -864,6 +913,14 @@ def runs_index(self): lo = (hit[0], None) self.live_overview[name] = lo o = lo[1] or o + # A child's results to act on decide whether its own page is closed: the summary its results.json + # reads counts them. When that summary or the overview above has one, that summary's overview + # replaces it, so the home page closes the child exactly when the child's page does, with its count. + # A child with none in either keeps the numbers it had without the gate. + if self.child(name, runs): + so = self.summary_overview(name, r["dir"] or os.path.join(self.root, name), st) + if so and any(((x or {}).get("levels") or {}).get("act") for x in (so, o)): + o = so item["attention"] = o["attention"] if o else None item["results"] = mtime is not None if o: # the Journal home: counts by level, the first findings, the lead @@ -871,8 +928,10 @@ def runs_index(self): screened=o.get("screened", False)) else: item["members"] = r.get("members") or [] + item["unreadable"] = bool(r.get("unreadable")) out.append(item) - fams = [{"name": x["name"], "url": x["url"], "members": x["members"]} for x in out if x["kind"] == "family"] + fams = [{"name": x["name"], "url": x["url"], "members": x["members"], "unreadable": x["unreadable"]} + for x in out if x["kind"] == "family"] return {"runs": [x for x in out if x["kind"] != "family"], "families": fams} diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 5c21a38e5..47725f49c 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -21,11 +21,14 @@ compound_het.tsv) with whether each child was assessed and why not (trio_status; a file linking out of the folder is "rejected" in trio_file_state), - each member's run progress and attention count, + each member's run progress, attention count and + whether their pages treat them as a child (child), and the members side by side: CYP2D6, the CPIC genes, HLA, the haplogroups and every blood group system, each value with its stale flag (from each - summary.json only) + summary.json only); 500 with a fixed line when a + file of the family cannot be read, and the family + then lists no members until it can state.json gets plugins.family: {"kind": "family", ...} on a family's page, {"kind": "member", "families": [...]} on the page of a sample in a family, @@ -481,23 +484,71 @@ def member_summary(live, d): return got if isinstance(got, dict) else None -def member_state(root, sample): - """The families that list the sample. A family whose pedigree cannot be read may be the one that - lists it as a child: with no readable family listing it, the answer is unknown, never "in no - family" ({"kind": "member", "families": [], "unknown": true}), and its pages keep results to act - on closed, as the home page does for a run no readable family lists.""" - fams, unread = [], False - for name, d in family_dirs(root): +_BUILT = {} + + +def folder_stamp(d): + """What a read of the family folder depends on: each entry's name, mode, size, mtime and ctime (a chmod changes + the ctime alone). None when the folder cannot be listed.""" + try: + names = sorted(os.listdir(d)) + except OSError: + return None + out = [] + for n in names: try: - order, _ = members_of(d) + st = os.stat(os.path.join(d, n)) + out.append((n, st.st_mode, st.st_size, st.st_mtime_ns, st.st_ctime_ns)) + except OSError: + out.append((n, None)) + return tuple(out) + + +def family_readable(name, d): + """Whether family.json's family part (family_page) builds from the folder's files as they are now, with its + pedigree read. Cached by folder_stamp, so a file that reads again counts at once, with no request needed.""" + stamp = folder_stamp(d) + hit = _BUILT.get(d) + if stamp is not None and hit and hit[0] == stamp: + return hit[1] + try: + ok = stamp is not None and not family_page(name, d)["state"].get("error") + except Exception: # any file of the folder that cannot be read + ok = False + _BUILT[d] = (stamp, ok) + return ok + + +def families(root): + """Each family folder under root as /runs.json lists it: {"name", "dir", "kind", "members", "unreadable"}. + members is [] while the pedigree cannot be read; unreadable while family.json cannot be built from the folder.""" + out = [] + for n, d in family_dirs(root): + try: + order, members = members_of(d) + people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"], + "mother": members[s]["mother"], "partner": members[s]["partner"], + "sequenced": members[s]["sequenced"]} for s in order] except (OSError, ValueError): - unread = True - continue - if sample in order: - fams.append({"name": name, "url": "/runs/%s/" % name}) - if fams: - return {"kind": "member", "families": fams} - return {"kind": "member", "families": [], "unknown": True} if unread else None + people = [] + out.append({"name": n, "dir": d, "kind": "family", "members": people, + "unreadable": not people or not family_readable(n, d)}) + return out + + +def member_state(root, sample): + """The families that list the sample, and child: whether its pages treat it as a child, by the one rule over + every family (bin/owngenome_live.py child_of). A family that cannot be read may be the one that lists it as a + child: with no readable family listing it, the answer is unknown, never "in no family" ({"kind": "member", + "families": [], "unknown": true}), and its pages keep results to act on closed, as the home page does.""" + from owngenome_live import child_of + fams = families(root) + child = child_of(fams, sample) + mine = [{"name": f["name"], "url": "/runs/%s/" % f["name"]} for f in fams + if any(m["sample"] == sample for m in f["members"])] + if mine: + return {"kind": "member", "families": mine, "child": child} + return {"kind": "member", "families": [], "unknown": True} if child else None def register(app): @@ -505,17 +556,7 @@ def root(): return os.path.realpath(app.config.root) def runs(app_): - out = [] - for n, d in family_dirs(root()): - try: - order, members = members_of(d) - people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"], - "mother": members[s]["mother"], "partner": members[s]["partner"], - "sequenced": members[s]["sequenced"]} for s in order] - except (OSError, ValueError): - people = [] - out.append({"name": n, "dir": d, "kind": "family", "members": people}) - return out + return families(root()) def page_data(req, m): name = m.group(1) @@ -523,19 +564,37 @@ def page_data(req, m): if not d or not is_family(d) or not inside(d, root()): app.log("family", name, "family.json", 404) return req.send_body(404, "text/plain; charset=utf-8", "not found\n") + try: + out = page_json(req, name, d) + except Exception as e: # an unreadable file is a fixed error, never a dropped connection + print(f"owngenome_live: family.json of {name} not built: {type(e).__name__}", file=sys.stderr) + app.log("family", name, "family.json", 500) + return req.send_body(500, "text/plain; charset=utf-8", "family not readable\n") + app.log("family", name, "family.json", 200) + return req.send_json(out) + + def page_json(req, name, d): out = family_page(name, d) live = getattr(req, "live", None) runs = {} + listed = live.runs() if live else {} for p in out["state"].get("members") or []: - r = live.run(p["sample"]) if live else None + r = listed.get(p["sample"]) if live else None if not r or r.get("kind") == "family": continue st = live.base_state(r) planned = [s for s in st["steps"] if s["state"] != "not planned"] - n, mtime = live.attention(r["dir"] or os.path.join(live.root, r["name"])) + rd = r["dir"] or os.path.join(live.root, r["name"]) + n, mtime = live.attention(rd) + # A child whose own page keeps the report closed has no count of flagged cards here: the count changes + # with which cards the unopened results flag, and a parent's page would show it. + if live.gated(p["sample"], rd, st, listed): + n = None + # child: whether the person's pages treat them as a child, by the one rule over every family (child_of), + # so the family page closes the same people as their own pages do. runs[p["sample"]] = {"url": "/runs/%s/" % p["sample"], "state": st["run"]["state"], "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"), - "attention": n, "results": mtime is not None} + "attention": n, "results": mtime is not None, "child": live.child(p["sample"], listed)} out["runs"] = runs given = {} comp_p = os.path.join(d, "compound_het.tsv") @@ -553,8 +612,7 @@ def page_data(req, m): r = live.run(s_) if live else None sums[s_] = member_summary(live, r["dir"] or os.path.join(live.root, r["name"])) if r else None out["side_by_side"] = side_by_side(seq, sums) if seq else None - app.log("family", name, "family.json", 200) - return req.send_json(out) + return out def hook(run_dir, state, events): if not run_dir: diff --git a/bin/owngenome_live_results.py b/bin/owngenome_live_results.py index e3698175b..e0caaf594 100644 --- a/bin/owngenome_live_results.py +++ b/bin/owngenome_live_results.py @@ -5,7 +5,9 @@ facts, its rows, its caveats and its attention flag, and the Journal's findings, levels, lead and ten sections GET /runs//finding// one finding's page (assets/live/finding.html): 404 unless - is a finding of that run's results.json now + is a finding of that run's results.json now; for a + child (child_run) with a result to act on, the page for + any well-formed id The summary is the one the server already reads for the step headlines (Live.summary): the run's summary.json when step 24 wrote it after every done @@ -42,6 +44,11 @@ def results(req, m): if summary: files = live.run_files(run) if run["dir"] else set() out = live_results.build(name, summary, files) + # For a child whose report the server keeps closed (family.json leaves its count out): the + # flagged-card count of summary.json, as family.json counts it otherwise. The family page shows it + # once every result to act on is opened there (assets/live/core.js released). + if live.gated(name, d, st): + out["saved_attention"] = live.attention(d)[0] else: out["note"] = "No step has finished yet: the results appear here as the steps end." except Exception as e: # a reader bug must not take the page down @@ -56,7 +63,10 @@ def results(req, m): def finding(req, m): """One finding's page. The page itself reads ../../results.json; this only checks that the id is - a finding of the run now, so a typed or stale address is a plain 404, never an empty page.""" + a finding of the run now, so a typed or stale address is a plain 404, never an empty page. A child with a + result to act on gets the page for any well-formed id: the page keeps the child's report closed + (assets/live/core.js) until every such result is opened, and a 404 would tell a guessed id apart from a + result the child has.""" live = req.live # A page: without a session it sends the browser to the login and back, as the viewer does. if live.password_hash and req.sid is None: @@ -70,7 +80,9 @@ def finding(req, m): try: d = run["dir"] or os.path.join(live.root, run["name"]) summary = live.summary(run["name"], d, live.base_state(run)) - ok = bool(summary) and any(f["id"] == fid for f in live_results_mod().build(run["name"], summary)["findings"]) + found = live_results_mod().build(run["name"], summary)["findings"] if summary else [] + ok = any(f["id"] == fid for f in found) or ( + any(f["level"] == "act" for f in found) and child_run(live, run["name"])) except Exception as e: # a reader bug is a 404 here, never a 500 print(f"owngenome_live: finding of {run['name']} not checked: {type(e).__name__}", file=sys.stderr) if not ok: @@ -87,6 +99,12 @@ def finding(req, m): app.route("GET", r"/runs/(%s)/finding/([^/]{1,200})/" % NAME, finding, auth=False) +def child_run(live, name): + """Whether the person page treats NAME as a child's run: the one rule of the page and the home page + (bin/owngenome_live.py child_of).""" + return live.child(name) + + def live_results_mod(): import live_results return live_results diff --git a/bin/render_report.py b/bin/render_report.py index 530cee487..9f73b1b41 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -53,7 +53,7 @@ def stale_note(sec): HLA_NOTE = ("HLA typing from short-read WGS is approximate. If either HLA-A or HLA-B allele has a T1K quality " "of 0 or less, that gene is not passed to PharmCAT (step 36).") EH_NOTE = ("Repeat-expansion calls from short reads can be wrong at some loci, and a long expansion " - "cannot be sized. Disease thresholds: docs/interpreting-results.md.") + "cannot be sized. Disease thresholds: docs/results/structural-and-repeats.md.") TELOMERE_UNIT = "intratelomeric reads per million reads with 48-52% GC" TELOMERE_NOTE = (f"Relative telomere content ({TELOMERE_UNIT}), not a telomere length: compare it only with " "samples of a similar age sequenced on the same platform (docs/10-telomere-analysis.md).") diff --git a/bin/run_eta.py b/bin/run_eta.py index bc407fd59..35ea90597 100755 --- a/bin/run_eta.py +++ b/bin/run_eta.py @@ -50,7 +50,7 @@ import run_state # noqa: E402 # Nextflow's default trace columns, in its order, then the ones the estimate needs. -# nextflow.config's trace.fields is the two lists joined; tests/test_run_eta.py checks it. +# nextflow.config's trace.fields is the two lists joined; rust/crates/og-live/tests/ops_eta.rs checks it. DEFAULT_FIELDS = ["task_id", "hash", "native_id", "name", "status", "exit", "submit", "duration", "realtime", "%cpu", "peak_rss", "peak_vmem", "rchar", "wchar"] ADDED_FIELDS = ["process", "tag", "attempt", "cpus", "memory", "start", "complete"] diff --git a/bin/run_headroom.py b/bin/run_headroom.py index 7714f7153..8c0ddc1ce 100755 --- a/bin/run_headroom.py +++ b/bin/run_headroom.py @@ -36,7 +36,7 @@ DEFAULT_WARN_PERCENT = 90 # The exits conf/base.config's errorStrategy retries: usually killed for memory or time. -# tests/test_run_headroom.py checks the two lists agree. +# og-runlog's MEMORY_EXITS is the same list; rust/crates/og-live/tests/ops_headroom.rs checks it against base.config. MEMORY_EXITS = {"143", "137", "104", "134", "139", "247"} DONE = {"COMPLETED", "FAILED", "ABORTED", "CACHED"} diff --git a/bin/str_intermediate_ranges.tsv b/bin/str_intermediate_ranges.tsv index f89587b18..fa834b5f0 100644 --- a/bin/str_intermediate_ranges.tsv +++ b/bin/str_intermediate_ranges.tsv @@ -5,7 +5,7 @@ # next_min: the first count of the next class, named in next_class; the range ends right below it. # label: what the report prints beside a count in the range. citation: the GeneReviews chapter that defines the range. # Only loci whose GeneReviews chapter defines such a range are listed. tests/test_str_ranges.py checks the rows -# and that docs/interpreting-results.md prints the same ranges. +# and that docs/results/structural-and-repeats.md prints the same ranges. locus intermediate_min intermediate_max next_min next_class label citation HTT 27 35 36 reduced penetrance intermediate (not disease-causing; may expand in offspring) GeneReviews: Huntington Disease, https://www.ncbi.nlm.nih.gov/books/NBK1305/ FMR1 45 54 55 premutation intermediate (gray zone; not disease-causing; may expand in offspring) GeneReviews: FMR1 Disorders, https://www.ncbi.nlm.nih.gov/books/NBK1384/ diff --git a/conf/containers.config b/conf/containers.config index f492cf6d2..7e8d7ecc7 100644 --- a/conf/containers.config +++ b/conf/containers.config @@ -16,7 +16,7 @@ process { withName: 'ARCHAIC_GENOTYPES' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } withName: 'ARCHAIC_IBDMIX' { container = 'quay.io/biocontainers/ibdmix:1.0.1--h4ac6f70_2' } withName: 'ARCHAIC_REPORT' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } - withName: 'BLOODTYPE' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } + withName: 'BLOODTYPE' { container = 'staphb/bcftools:1.24' } withName: 'BLOODTYPE_DEPTH' { container = 'staphb/samtools:1.24' } withName: 'BLOODTYPE_REGIONS' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } withName: 'CHIP_ANNOTATE' { container = 'staphb/bcftools:1.24' } diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index dfa88ad7d..b6baa03d2 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -4,6 +4,22 @@ One-time downloads required before running the pipeline. Each heading below give > **Estimated time:** 1-3 hours depending on internet speed. The two VEP caches (26 GB for step 13, 24 GB for step 17) are the largest downloads of a default run. +A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. +These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. +Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 15 GB, and VEP 115 needs 52 GB. +Each check adds a 5 GB margin and credits the disk space occupied by a partial download. +Stale extraction files are removed before the check. Retained PCGR archives are checked against the pinned SHA-256 before reuse. An item that does not fit is skipped with a warning. Setup continues. + +```bash +./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads +./scripts/setup.sh --clinical-data "$GENOME_DIR" # install or retry only these three +``` + +An interrupted transfer resumes. Existing caches and bundles are kept. +`--clinical-data` exits 1 when any of the three is still missing after it runs. +Without these data, `run-all.sh` skips the affected steps and records the reason. +A bare `nextflow run` still needs its tools and data parameters. Setup does not change that selection. + ## GRCh38 Reference Genome The foundation for everything. All tools need this. The pipeline uses NCBI's **GRCh38 no-ALT analysis set**: chr1-22, X, Y and M, the unplaced and unlocalized scaffolds and the EBV genome, 195 sequences, with UCSC names (`chr1`, `chrM`). `setup.sh` downloads it, checks it and stores it as `reference/GRCh38_no_alt_analysis_set.fasta`: @@ -143,7 +159,7 @@ Required for step 5. `setup.sh` downloads them (about 5 GB, about 20 GB unpacked ## VEP Cache (~26 GB) -Ensembl Variant Effect Predictor annotation database. Required for step 13 and the pipeline's VEP. Without it, a run skips VEP, vcfanno, the clinical filter and slivar. Install it before the first run: +Ensembl Variant Effect Predictor annotation database. Required for step 13 and the pipeline's VEP. Setup installs it by default. Without it, a run skips VEP, vcfanno, the clinical filter and slivar. To install only this cache: ```bash ./scripts/setup.sh --vep-cache ${GENOME_DIR} @@ -178,38 +194,22 @@ Required for step 17 (CPSR cancer predisposition screening). Includes ClinVar, g > **Important:** PCGR 2.3.2 bundles VEP 115, which requires a **release-115** cache — different from the release-116 cache used by step 13 above. See the next section for the VEP 115 download. The bundle date (`PCGR_DATA_BUNDLE`) and the cache release (`PCGR_VEP_CACHE_RELEASE`) are in `versions.env` and move with `PCGR_IMAGE`. -```bash -mkdir -p ${GENOME_DIR}/pcgr_data -cd ${GENOME_DIR}/pcgr_data - -# Download (~7 GB) -wget -c https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.20260620.grch38.tgz +Setup installs it by default. To add the clinical data to an existing setup: -# Extract and organize into version-stamped directory -tar xzf pcgr_ref_data.20260620.grch38.tgz -mkdir -p 20260620 && mv data/ 20260620/ -# Creates: ${GENOME_DIR}/pcgr_data/20260620/data/ - -# Optional: delete the tarball to save 7 GB -# rm pcgr_ref_data.20260620.grch38.tgz +```bash +./scripts/setup.sh --clinical-data "$GENOME_DIR" ``` +The installer checks the archive against `PCGR_DATA_SHA256` in `versions.env`, then extracts it into a temporary folder. +The server publishes no checksum, so the size (`PCGR_DATA_BYTES`) and the digest are measured from a full download. `PCGR_DATA_MEASURED_FOR` names the image tag and bundle date they belong to, and `scripts/ci/check-coupled-versions.sh` fails when `PCGR_IMAGE` or `PCGR_DATA_BUNDLE` moves without it. +Only a complete tree is moved to `pcgr_data/20260620/data/`. The archive is deleted after installation. +The [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) names the bundle's third-party databases. Their own terms apply. Setup prints the resource link before downloading. + ## VEP 115 Cache for CPSR (~24 GB) PCGR 2.3.2 (step 17) bundles VEP 115 internally, which needs the **release-115** cache. This is separate from the release-116 cache used by step 13. Both coexist in the same `vep_cache/` directory under different subdirectories (`116_GRCh38/` and `115_GRCh38/`). A `113_GRCh38/` cache left from PCGR 2.2.5 is no longer read and can be deleted. -```bash -mkdir -p ${GENOME_DIR}/vep_cache/tmp -cd ${GENOME_DIR}/vep_cache/tmp - -# Download VEP 115 cache (~24 GB) -wget -c https://ftp.ensembl.org/pub/release-115/variation/indexed_vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz - -# Extract alongside the existing release-116 cache -cd ${GENOME_DIR}/vep_cache -tar xzf tmp/homo_sapiens_vep_115_GRCh38.tar.gz -# Creates: ${GENOME_DIR}/vep_cache/homo_sapiens/115_GRCh38/ -``` +Setup installs this cache by default, after the PCGR bundle. `--clinical-data` retries it using Ensembl's `CHECKSUMS` file and a temporary extraction folder. > If you only run step 13 (VEP annotation) and skip step 17 (CPSR), you only need the release-116 cache. If you only run step 17, you only need release-115. @@ -510,7 +510,7 @@ wget -c https://giab.s3.amazonaws.com/release/AshkenazimTrio/HG002_NA24385_son/N [Hardware and storage requirements](hardware-requirements.md#shared-reference-data-one-time) has the one table of every download above, with the totals. -> **Tip:** If disk space is tight, you can skip the VEP cache (step 13) and PCGR bundle (step 17) initially. The core pipeline (steps 2-3-6-7) only needs the reference FASTA and ClinVar (~3.5 GB unpacked). +> **Tip:** If disk space is tight, use `setup.sh --no-clinical-data` to skip both VEP caches and the PCGR bundle initially. The core pipeline (steps 2-3-6-7) only needs the reference FASTA and ClinVar (~3.5 GB unpacked). ## Verifying Your Setup diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index 3a8e5873a..b1292af01 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -51,14 +51,14 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI 1. Lists the contigs each file holds records on (`bcftools index -s`) and keeps those the reference has too: ClinVar's contigs that are in the reference, and the sample's contigs among those. `bcftools norm` stops at the first record whose contig the reference lacks, such as a provider's unplaced scaffold or one of the `NT_` contigs the full ClinVar file has, and a record on a contig the other file lacks cannot match anyway. The step prints how many of the sample's records it leaves out and on which contigs. When no contig is shared it stops with an error instead of reporting zero hits, and says which file is named the other way (`1, 2, ...` against `chr1, chr2, ...`). 2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. -4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. +4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. A sample record whose REF is not the reference's base at that position is kept as written (`bcftools norm -c w`), with a `REF_MISMATCH` warning in the log. Such a record cannot match a ClinVar allele, so the summary of item 11 counts these records, and prints a notice to check that the VCF was called on this reference when there are any. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. 6. Keeps the sample's records and combined alleles whose allele is in ClinVar (`bcftools isec -n=2 -w1`), copies ClinVar's `ID`, `GENEINFO`, `CLNSIG`, `CLNREVSTAT`, `CLNDN` and `CLNDISDB` onto them (`bcftools annotate --pair-logic exact`), and keeps only those whose genotype carries an ALT allele (`bcftools view -i 'GT~"[1-9]"'`, a non-zero allele index anywhere in the genotype). The match is by allele, so without this a `0/0` or `./.` record, the `0/0` half of a split multiallelic record and a reference-only row with ALT `.` would be listed as hits. A half call such as `./1` carries the ALT allele and is a hit; `GT="alt"` would drop it. The filter runs after the split, where it sees each allele's own genotype. No command line goes into the hits file's header (`--no-version`), so ClinVar's path on your machine stays out of it. `CLNDN` is the condition ClinVar names for the allele and `CLNDISDB` its ids in disease databases such as MONDO, OMIM and MedGen. A gene can cause more than one disease, so a later step reads these to know which one ClinVar asserted. Every ClinVar release defines both. `bcftools annotate` stops on a tag the ClinVar file does not define, so a hand-built ClinVar file without them gives hits without them, and the step prints a notice. 7. Marks the hits that belong together. A matched combined allele is a hit of its own and carries `INFO/PGP_JOINED_FROM`, its two parts as `CHROM:POS:REF:ALT`. Each of its parts that is a hit too keeps its row and gets `INFO/PGP_PART_OF`, the combined record it belongs to, so the reader sees one allele with the combined record's own review status, not two unrelated hits. Two hits in one gene that could be one allele by the rule of item 5 but matched no combined record each get `INFO/PGP_ONE_ALLELE`, the other hit: possibly one allele, check ClinVar for a combined record. 8. Writes a call floor other than the default into the hits file's header, for example `##PGP_CALL_FLOOR=GQ=20,DP=8` (below). At the default the file has no such line and reads as GQ 30 and DP 10. 9. Writes the always-report list (below): the sample's call at every site of `assets/deep_intronic_sites.tsv`. 10. Runs the cDNA read check on each hit and the cDNA scan of every exon (below). -11. Prints the hits grouped by ClinVar's review stars (`og clinvar hits`): 4 practice guideline, 3 expert panel, 2 multiple submitters with no conflict, 1 a single submitter or conflicting classifications, 0 no assertion criteria, each with its combined-allele note when it has one and "LOW-CONFIDENCE CALL (GQ .., DP ..): check the reads" when the call is below the floor, and the count of low-confidence calls. The screening file keeps every Pathogenic/Likely_pathogenic submission whatever its review status, so a zero-star hit counts as a hit; the stars say how much weight it deserves. The Nextflow module runs the same steps and prints the same counts; its `CLINVAR_CDNA_CHECK` process runs the cDNA read check and the cDNA scan. +11. Prints the hits grouped by ClinVar's review stars (`og clinvar hits`): 4 practice guideline, 3 expert panel, 2 multiple submitters with no conflict, 1 a single submitter or conflicting classifications, 0 no assertion criteria, each with its combined-allele note when it has one and "LOW-CONFIDENCE CALL (GQ .., DP ..): check the reads" when the call is below the floor, and the count of low-confidence calls, then the count of `REF_MISMATCH` records of item 4. The screening file keeps every Pathogenic/Likely_pathogenic submission whatever its review status, so a zero-star hit counts as a hit; the stars say how much weight it deserves. The Nextflow module runs the same steps and prints the same counts; its `CLINVAR_CDNA_CHECK` process runs the cDNA read check and the cDNA scan. A delins that the caller writes as two records, while ClinVar holds it as one, is matched exactly once, as the combined record of item 5: `tests/test_clinvar_join_adjacent.py` check 8 runs the real step on such a fixture and counts the hits. @@ -66,7 +66,18 @@ A delins that the caller writes as two records, while ClinVar holds it as one, i Some pathogenic alleles sit deep inside an intron, far from any exon. Filters built around exons and splice sites skip them, and a report that lists only hits cannot say whether such a site was read as the reference or not read at all ([Vaz-Drago 2017](https://pubmed.ncbi.nlm.nih.gov/28497172/)). `assets/deep_intronic_sites.tsv` lists the best known ones: CFTR c.3718-2477C>T and c.1680-886A>G, CEP290 c.2991+1655A>G, USH2A c.7595-2144A>G, ATM c.5763-1050A>G and GLA c.640-801G>A, each a ClinVar Pathogenic record with an expert-panel or multi-submitter review, at the GRCh38 position and alleles of ClinVar's canonical SPDI (checked 2026-11-04). -Every run writes `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv`, one row per site, carrier or not. Its `call` is the genotype in the words the hits use (het, hom, ...) when the sample carries the site's allele; `ref` for a reference call, with the GQ of the `0/0` record or of the gVCF reference block over the site (`ref (low confidence)` when that GQ is below the call floor, `CLINVAR_MIN_GQ` or `ext.min_gq`, default 30); `other allele` when the sample carries another allele there; or `no call`, with the reason (no record in the VCF or the gVCF, a gVCF record that is not a reference call, or no gVCF at all). `detail` holds the GQ and DP, the other allele or the reason. The records read are the split PASS records of item 4 at the site's position (`bcftools query -T`) and the gVCF records over it (`bcftools query -R`). Both reports print every row under the ClinVar hits. +Every run writes `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv`, one row per site, carrier or not. The records read are the split PASS records of item 4 at the site's position (`bcftools query -T`) and the gVCF records over it (`bcftools query -R`), each with its GT, GQ, DP and FILTER. The `call` column is one of these: + +| `call` | When | +|---|---| +| a genotype (`het`, `hom`, ...) | The sample carries the site's allele, in the words the hits use. One copy on chrX or chrY outside the pseudoautosomal regions (GT `1`, as some callers write a male's X) reads `hemizygous`. | +| `ref` | A reference call: a record whose alleles are all `0` (`0/0`, or `0` on chrX or chrY outside the pseudoautosomal regions), or a gVCF reference block over the site. Its FILTER is `PASS`, `.` or `RefCall` (DeepVariant's reference call), its GQ is at or above the call floor (`CLINVAR_MIN_GQ` or `ext.min_gq`, default 30; a floor set below 20 counts as 20 here, so a lower floor changes the hits only), and its depth is at or above the depth floor (`CLINVAR_MIN_DP` or `ext.min_dp`, default 10). | +| `ref (low confidence)` | A reference call with any other FILTER (`LowQual`, for example), a GQ below the call floor, or no GQ at all, so nothing checked it against the floor. | +| `ref (shallow)` | A reference call whose only fault is a depth (`FORMAT/DP`) below the depth floor. A DeepVariant reference block has no DP (it writes `MIN_DP`), and its GQ already falls with the depth, so only its GQ is checked. | +| `other allele` | The sample carries another allele at the site. | +| `no call` | Nothing says reference or carrier: no record in the VCF or the gVCF, no gVCF at all, a record with no genotype (`./.`), a record that is not a reference call (`0/.`, or a filtered call of the allele that only the gVCF holds), or a one-copy GT where the genome has two copies (an autosome, or chrX inside a pseudoautosomal region). A record for another allele at the site that gives none of these is named the same way, with its alleles. | + +`detail` holds the call's GQ and DP (`GQ n (gVCF block)` for a block, with its DP when that is below the depth floor), each reason a reference call is not plain `ref` (`FILTER LowQual`, `below the call floor of GQ 30`, `no GQ to check against the call floor of GQ 30`, `below the depth floor of DP 10`), the other allele, or what the record at the site holds when there is no call. A no call is not a negative result: it says only that the site was not read. These floors apply to reference calls only. A carrier's GQ and DP are shown, not judged: the same allele is a ClinVar hit, and the hits table marks it low confidence when it is below the floor. Both reports print every row under the ClinVar hits. ### Lab-construct sites and the cDNA scan diff --git a/docs/07-pharmacogenomics.md b/docs/07-pharmacogenomics.md index 49641c30a..3c50af7ca 100644 --- a/docs/07-pharmacogenomics.md +++ b/docs/07-pharmacogenomics.md @@ -97,7 +97,7 @@ docker run --rm \ - **gVCF.** PharmCAT refuses a gVCF, and decides by the file name too (`.g.vcf`, `.genomic.vcf`). Yet a gVCF is the better input: a variants-only VCF leaves about half of PharmCAT's genes Unknown, because PharmCAT cannot tell a reference call from a position that was not covered. So the script reads `vcf/${SAMPLE}.g.vcf.gz` when step 3 wrote one and expands its reference blocks into `${SAMPLE}.pgx_regions.vcf.gz` (step 0 above, deleted afterwards), which PharmCAT accepts; without a gVCF it reads `${SAMPLE}.vcf.gz`, and does not check whether that file is itself a gVCF (PharmCAT then stops on it). The Nextflow pipeline does the same with DeepVariant's gVCF, or with the gVCF a samplesheet row gives in its `gvcf` column (`run-all.sh` fills it from `vcf/${SAMPLE}.g.vcf.gz`); a gVCF given in the `vcf` column stops the run before any analysis when `pharmcat` is selected. For a vendor gVCF, remove the reference blocks and rename the file: [Starting from a Vendor VCF](vcf-first.md). - **Positions longer than one base.** The expansion writes one record per base, each with a one-base REF. 51 PharmCAT 3.4.0 positions have a REF or ALT longer than one base: the insertions and deletions of DPYD, UGT1A1 (\*28, the TA-repeat insertion at chr2:233760233), TPMT, CYP3A4, CYP3A5, CFTR, CYP2C9, NUDT15, RYR1, CYP2D6 and G6PD, and two two-base substitutions in RYR1. PharmCAT's preprocessor does not take a one-base record as a call at such a position: it sets the record aside (FILTER `PCATxINDEL`, or `PCATxREF` for the substitutions) and lists the position in `missing_pgx_var.vcf`. Where PharmCAT's REF is longer than one base (the deletions, UGT1A1 \*28, the substitutions), its matcher then discards the record ("REF in VCF (C) does not match expected reference (CAT)"), so the position has no call and the alleles defined there are uncallable, even where the gVCF shows you match the reference over the whole span. At an insertion with a one-base REF the matcher still reads the set-aside record as a reference call ("PharmCAT cannot validate this position"). So step 0 runs `og pgx refblock-fill` on the expanded file. For each of these positions in PharmCAT's positions file (`/pharmcat/pharmcat_positions.vcf.bgz` in the image), it writes a 0/0 record with PharmCAT's own ID, REF and ALT (INFO flag `PGX_REFBLOCK`) in place of the one-base record, when every base of the REF span lies in reference blocks with GQ 20 or more (GQ is Phred-scaled: 20 means a 1 in 100 chance the reference call is wrong; genotype 0/0, or 0 on a haploid chrX; FILTER `PASS` or `.`) and no other record overlaps the span: no variant call, no `./.` block, no block below GQ 20. Otherwise the expanded file stays as it was there: the position stays missing, and at an insertion the matcher reads the one-base record as before. The step log names the positions it filled (`pgx_refblock_fill: N of 51 PharmCAT positions longer than one base called 0/0 from reference blocks with GQ >= 20: ...`). The GQ threshold applies to these positions only: a one-base position takes any 0/0 block, as before. Where the expanded file has a record at such a position that is not a one-base reference call, that record stays and no 0/0 record is added beside it; the log says so. The Nextflow module does the same. -- **MT-RNR1 from the chrM calls.** PharmCAT 3.4.0 has no MT-RNR1 definition for its matcher: it reads the gene only as an outside call, one allele (chrM is not a pair), so from a VCF alone MT-RNR1 is No Result on every sample. When [step 20](20-mtoolbox.md) wrote its Mutect2 chrM calls (`mito/${SAMPLE}_chrM_filtered.vcf.gz`), the script runs `og pgx mito-merge` (step 3 above). It reads PharmCAT's own MT-RNR1 alleles from the jar (`Reference` and names such as `m.1555A>G`) and adds one MT-RNR1 line to a copy of step 36's outside calls (`${SAMPLE}.pharmcat_outside_calls.tsv`, deleted afterwards): a named single-base allele with FILTER `PASS` and an allele fraction (AF) of 0.95 or more, the level the pipeline calls homoplasmic; the one PharmCAT ranks highest (increased risk, then uncertain, then normal) when there are several, with the others named in the step log; or `Reference` when no named allele is called. Mutect2 writes variant records only, so a position without reads has no record either: `Reference` needs read evidence, 20 reads or more (`samtools depth`, MAPQ and base quality 20) at every position where PharmCAT names an MT-RNR1 allele, from the sample's BAM (`aligned/${SAMPLE}_sorted.bam`, the one step 20 read). Without the BAM, or with a position below 20 reads, no `Reference` is passed and the log names the positions; a homoplasmic named allele needs no depth, its record is the evidence. Mutect2 writes the genotype 0/1 even at AF 1.0, so the AF decides, not the genotype. No line is added, and the gene stays No Result with the reason in the step log, when a named allele is heteroplasmic (AF below 0.95), filtered, or has no AF; when a homoplasmic `PASS` record at a position where PharmCAT names an allele (m.960 and m.961 included, named there only as deletions) has an ALT PharmCAT does not name; when a `PASS` insertion or deletion overlaps a named position or lies within 10 bases of m.960 or m.961, where PharmCAT names deletions in a C run that the script does not map; and when the file has no `PASS` chrM record. Without step 20's file the step says so and MT-RNR1 stays No Result: run step 20 before this step. The PHARMCAT module does the same with MITO_VARIANTS' calls. +- **MT-RNR1 from the chrM calls.** PharmCAT 3.4.0 has no MT-RNR1 definition for its matcher: it reads the gene only as an outside call, one allele (chrM is not a pair), so from a VCF alone MT-RNR1 is No Result on every sample. When [step 20](20-mtoolbox.md) wrote its Mutect2 chrM calls (`mito/${SAMPLE}_chrM_filtered.vcf.gz`), the script runs `og pgx mito-merge` (step 3 above). It reads PharmCAT's own MT-RNR1 alleles from the jar (`Reference` and names such as `m.1555A>G`) and adds one MT-RNR1 line to a copy of step 36's outside calls (`${SAMPLE}.pharmcat_outside_calls.tsv`, deleted afterwards): a named single-base allele with FILTER `PASS` and an allele fraction (AF) of 0.05 or more, the pipeline's heteroplasmy floor; the one PharmCAT ranks highest (increased risk, then uncertain, then normal) when there are several, with the others named in the step log; or `Reference` when no named allele is called. A heteroplasmic allele (AF 0.05 to 0.95) is passed like a homoplasmic one: CPIC's MT-RNR1 guideline sets no heteroplasmy level at which aminoglycosides become safe, and asks for the homoplasmic guidance whenever a relevant variant is detected. So a heteroplasmic m.1555A>G or m.1494C>T gives PharmCAT's aminoglycoside recommendation, and the step log says `heteroplasmic` with the AF (the [step 20](20-mtoolbox.md) table and the report's chrM section give the level too). Mutect2 writes variant records only, so a position without reads has no record either: `Reference` needs read evidence, 20 reads or more (`samtools depth`, MAPQ and base quality 20) at every position where PharmCAT names an MT-RNR1 allele, from the sample's BAM (`aligned/${SAMPLE}_sorted.bam`, the one step 20 read). Without the BAM, or with a position below 20 reads, no `Reference` is passed and the log names the positions; a named allele needs no depth, its record is the evidence. Mutect2 writes the genotype 0/1 even at AF 1.0, so the AF decides, not the genotype. No line is added, and the gene stays No Result with the reason in the step log, when a named allele is below the detection floor (AF below 0.05) and no other named allele is called (the log says so, and the gene is not called Reference either; beside a called allele it is only named in the log), filtered, or has no AF (or an AF that reads `nan`); when a `PASS` record at AF 0.05 or more at a position where PharmCAT names an allele (m.960 and m.961 included, named there only as deletions) has an ALT PharmCAT does not name; when a `PASS` insertion or deletion overlaps a named position or lies within 10 bases of m.960 or m.961, where PharmCAT names deletions in a C run that the script does not map; and when the file has no `PASS` chrM record. Without step 20's file the step says so and MT-RNR1 stays No Result: run step 20 before this step. The PHARMCAT module does the same with MITO_VARIANTS' calls. - **PharmCAT's own log file.** The jar's built-in `logback.xml` writes `pharmcat.log` into the working directory. In the image that is `/pharmcat`, which the calling user cannot write, so every run printed a logback status dump with a `pharmcat.log (Permission denied)` stack trace (the calls were not affected). The script runs the jar with `-w /tmp`, a writable directory that goes away with the container. A Nextflow task runs in its own writable work directory, so the module needs nothing: the log lands there. - **A backslash in a `##` header line.** PharmCAT up to 3.4.0 bundles vcf-parser 0.3.1, which stops with "Error parsing metadata: character to be escaped is missing" on one. The line is valid VCF; bcftools writes it for a soft filter with a quoted string (`bcftools filter -s LowDP -e 'FORMAT/DP<10 && GT!="0/0"'`). The script and the Nextflow module rewrite the header of PharmCAT's own copy (`${SAMPLE}.pharmcat_input.vcf`, deleted afterwards by the script): on `##` lines `\"` becomes `'` and any other `\` becomes `/`. PharmCAT's calls are the same with and without the rewrite. A newer PharmCAT is no fix yet: 3.4.0 still bundles vcf-parser 0.3.1 and fails the same way. @@ -120,7 +120,7 @@ docker run --rm \ ## Limitations - **HLA-A, HLA-B and CYP2D6** are not called from a VCF: PharmCAT 3.4.0 reports them with no result (`callSource` `NONE`). [Step 36](36-pgx-consensus.md) gives PharmCAT T1K's HLA types (step 8) and a CYP2D6 call only when pypgx (step 32) and Cyrius (step 21, opt-in) agree on depth that passed its check; PharmCAT reads them with `-po` (`callSource` `OUTSIDE`). Run step 36 before this step, or run this step again after it. A disagreement, or one caller alone, leaves CYP2D6 without a result, on purpose. -- **MT-RNR1** is an outside call from step 20's chrM calls (see above): PharmCAT 3.4.0 does not call it from a VCF. A heteroplasmic named allele, such as m.1555A>G at AF 0.4, is not passed, so the gene stays No Result and the step log names the allele and its AF: read it there. +- **MT-RNR1** is an outside call from step 20's chrM calls (see above): PharmCAT 3.4.0 does not call it from a VCF. A heteroplasmic named allele, such as m.1555A>G at AF 0.4, is passed and gets the aminoglycoside recommendation; the PharmCAT report does not show its AF, the step log does. A named allele below AF 0.05 is not passed. With no other named allele called, the gene stays No Result and the step log names the allele and its AF as below the detection floor. At that level the call cannot be told from noise or NuMT reads, so it is not called absent either. - **CYP4F2 can come out No Result** when unphased heterozygous calls elsewhere in the gene, a weak one among them, leave no diplotype that matches. CPIC's warfarin guidance needs only the \*3 position (rs2108622). Step 27 then reads the genotype PharmCAT itself lists at that position in `report.json` (the gene's `variants`) and prints it in a section headed "Single-Position Fallback (not a PharmCAT call)", with the CPIC warfarin rule when you carry the variant. It does the same for VKORC1 at rs9923231 (-1639G>A). PharmCAT's own result for the gene stays No Result in the phenotypes table. A position with no genotype is reported as not callable. The report carries no GQ or read depth for the position: check it in your VCF before acting on it. - PharmCAT may disagree with lab reports on complex haplotypes (e.g., NAT2). Discrepancies can arise from different genome builds (hg19 vs hg38), different star allele definitions, or different variant calling pipelines. When a discrepancy matters clinically, compare both sets of raw variant calls and consult the PharmVar database for the current allele definitions — do not blindly trust either source. - PharmCAT output structure changes across releases. If you upgrade PharmCAT, re-test step 27 (`27-cpic-lookup.sh`) because it parses the JSON output directly. diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 20a8146bb..c9fe48202 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -42,7 +42,7 @@ For chip data (no `FORMAT/PL`), the script adds `-G30`. ### Centromere and heterochromatin bands -Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `og summary collect` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. +Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10). Neighbouring rows are merged around each centromere, with two spans on chrY. `MASKED_BANDS` in `rust/crates/og-summary/src/collect/group_c/roh.rs` is their only copy. The step and the Nextflow module call `og roh summary`, which reads that same constant. On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. The reports count autosomes only: Segments, Total and Largest keep chr1 to chr22, so they leave out chrX and chrY (a man's single X reads as one long run) and any other contig (chrM, chrEBV, chrUn_*), and the part of each segment in a band. They say how many chrX and chrY segments and how many megabases in bands were left out. @@ -63,8 +63,15 @@ When parents are related, a child is autozygous (both copies from the same ances These are averages. Inheritance is random, so one person's total can be well above or below the value for their parents' relationship, and the ranges of neighbouring rows overlap. A total near one row is consistent with it; it does not prove it. -Both reports print this as **F_ROH**, the measure of McQuillan et al. 2008 (PMID 18760389): the length of the autosomal segments of 5 Mb or more, without their band parts, divided by the length of chr1 to chr22 without the bands (2,875,001,522 bp less 217,000,000 bp, so 2,658,001,522 bp). Beside it they name the row of the table above whose F is nearest to F_ROH on a log scale: the boundaries are the geometric means of neighbouring rows (for example 1/16 and 1/8 meet at about 0.088), and an F_ROH under 1/128 (about 21 Mb) reads "no sign of related parents". When one chromosome carries most of the runs (next section), there is no band: that chromosome fills F_ROH by itself. +`og summary collect` writes **F_ROH**, the measure of McQuillan et al. 2008 ([PMID 18760389](https://pubmed.ncbi.nlm.nih.gov/18760389/)). Its numerator is the length of autosomal runs of 5 Mb or more outside the bands. Its denominator is the GRCh38 length of the chromosomes this run assessed, with their bands removed. With all 22 autosomes, that is 2,875,001,522 bp less 217,000,000 bp, or 2,658,001,522 bp. With chr20 alone it is 64,444,167 bp less 4,700,000 bp, or 59,744,167 bp. A 21 Mb run then gives F_ROH 0.3515, rather than 0.0079. +The collector streams the per-site `ST` rows to identify assessed autosomes. A chromosome with sites counts even when it has no ROH. No marker-count floor is imposed, so sparse exome and chip input remain supported. This is a chromosome-length denominator, **not a count of callable bases**. One site does not establish coverage across a chromosome. A capture panel or a small interval on every autosome can still look complete at this resolution. + +The new `roh.data.scope` object records `autosomes` in numeric order, `unmasked_bp`, `source`, `partial` and a note. Older segment-only files have `source: segments_only`; their observed chromosomes give only a lower bound on scope. An empty file has no denominator and `f_roh: null`. The segment list and length totals remain available. + +A relationship label or a one-chromosome UPD flag needs sites on all 22 autosomes and the full 2,658,001,522 bp denominator. Below that floor, or without per-site evidence, `relationship` and `upd_flag` are null. A subset F_ROH is a descriptive fraction of the listed chromosomes and must not be mapped to the parental relationship table. The report and live-view wording for this scope is handled separately. + +For complete chromosome scope, the relationship is the row whose F is nearest on a log scale. Boundaries are the geometric means of neighbouring rows; for example 1/16 and 1/8 meet at about 0.088. An F_ROH under 1/128 reads "no sign of related parents". A one-chromosome flag suppresses that relationship label. To compute the total the table means from the summary file, which takes the band parts out before it keeps the segments of 5 Mb or more, as the reports do: ```bash @@ -78,24 +85,19 @@ In uniparental disomy (UPD) both copies of a chromosome come from one parent. Wh The reports flag a chromosome when its segments of 5 Mb or more (band parts left out) add up to at least 20 Mb **and** to more than half of the length of all such autosomal segments. Both numbers are our thresholds, not a published cut-off: 20 Mb on one chromosome is the long-standing sign of isodisomy (see the uniparental disomy note under [Important Notes](#important-notes)), and the share rule keeps out a child of related parents, whose runs spread over many chromosomes. The flag says "may": a run that long can also be chance in a child of related parents. -A flagged chromosome is looked up in `assets/imprinted_regions.tsv`, the chromosomes where UPD from one parent causes an imprinting disorder (overview: Eggermann et al. 2023, Nat Rev Dis Primers, PMID 37386011). A row with a region applies only when a run on that chromosome overlaps it; the others apply to the whole chromosome. +A flagged chromosome is looked up in [assets/imprinted_regions.tsv](https://github.com/GeiserX/OwnGenome/blob/main/assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. -| Chromosome or region | UPD from the mother | UPD from the father | Source (PMID) | -|---|---|---|---| -| chr6 (6q24, PLAGL1, HYMAI) | none known | Transient neonatal diabetes mellitus | 20301706 | -| chr7 (GRB10 at 7p12, MEST at 7q32) | Silver-Russell syndrome | none known | 27585961 | -| 11p15, chr11:0-22,000,000 (H19/IGF2, KCNQ1OT1) | Silver-Russell syndrome | Beckwith-Wiedemann syndrome | 29377879, 27585961 | -| chr14 (14q32.2, DLK1-MEG3) | Temple syndrome | Kagami-Ogata syndrome | 24891339, 26377239 | -| 15q11-q13, chr15:20,500,000-33,400,000 (SNRPN, UBE3A) | Prader-Willi syndrome | Angelman syndrome | 22237428, 27615419 | -| chr20 (20q13.32, GNAS) | Mulchandani-Bhoj-Conlin syndrome (growth failure) | Pseudohypoparathyroidism type 1B | 26248010, 29959430 | +The flag lists the chromosome, run length, share, region names and citations. It names no syndrome. The existing `maternal_upd` and `paternal_upd` JSON keys stay as `"-"` for reader compatibility. The asset retains their reference facts. Those facts cannot be inferred from a region match in one genome. -`og summary collect` holds the same rows (`IMPRINTED_REGIONS` in `rust/crates/og-summary`; a unit test checks that the two agree). A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. +All six chromosomes use the same chromosome-wide lookup, including chr11 and chr15. Mixed iso/heterodisomy can leave an imprinted locus outside the observed homozygous runs ([Robinson 2000](https://pubmed.ncbi.nlm.nih.gov/10797485/)). A region hint therefore does not claim that a run overlaps the imprinted locus. Region names are broad cytoband labels, not diagnostic locus intervals or a specialist-reviewed assay. No interval was narrowed on the basis of a ROH alone. -What this cannot tell, and the reports say so: +The flag's `note` describes long chromosome-focused ROH and suggests investigating possible isodisomy. Its limits are: -- **Which parent.** A single genome has no parent of origin; both columns are listed. The parents' genotypes (step 38, the family workflow) confirm a UPD and tell which. Where a row applies, a methylation test of its imprinted region tests for that imprinting disorder; on its own it does not prove a UPD, since an imprinting defect gives the same result. -- **Heterodisomy.** Two different copies from one parent leave no run at all, so it is not detected here. It needs the parents' genotypes. -- **Mosaic or segmental UPD** that covers only a short stretch, or a share of the cells (common in Beckwith-Wiedemann syndrome), may stay under the thresholds. +- A large deletion of one copy or acquired loss of heterozygosity can give the same pattern. Check the copy-number results and the MoChA results. The collector does not establish copy neutrality or perform that cross-check. +- Shared ancestry, including related parents, can also produce long runs. A flag is a screening hint, not a UPD diagnosis. +- A single genome cannot determine parental origin. The parents' genotypes can establish it. Methylation testing assesses an imprinting defect and alone does not establish UPD. +- Heterodisomy can leave no ROH. Mosaic or short segmental UPD can fall below the thresholds. +- Sites on all autosomes establish chromosome presence only. They do not prove assay completeness within any chromosome. ### Single segments @@ -112,5 +114,5 @@ What this cannot tell, and the reports say so: - bcftools roh reads a site's frequency from INFO/AC and INFO/AN when the record has them, and uses 0.4 only where it does not. DeepVariant writes neither, so on its calls every site uses 0.4. A VCF from another caller may carry them: in a single-sample file they are the sample's own allele counts, not a population's. - No population allele frequencies are used, and the reports say so: read the total and the short segments as rough, the long segments are the signal. A population allele-frequency file (`--AF-file`, for example from gnomAD) would replace the one default value with real frequencies per site and give better segment edges; neither the script nor the module uses one today, and its effect on the total has not been measured. - The centromere and heterochromatin bands (above) are taken out of every count; the step and the reports do it, so there is nothing to subtract by hand -- Long runs concentrated on one chromosome may indicate uniparental disomy; the reports flag a chromosome whose segments of 5 Mb or more add up to at least 20 Mb and to more than half of the length of all such autosomal segments (above). Confirm with the parents' genotypes; a methylation test of a listed imprinted region tests for its imprinting disorder +- Long runs concentrated on one chromosome may indicate uniparental disomy; the reports flag a chromosome whose segments of 5 Mb or more add up to at least 20 Mb and to more than half of the length of all such autosomal segments (above). The flag requires sites on all 22 autosomes. Check copy number and acquired LOH, and confirm with the parents' genotypes. A methylation test assesses an imprinting defect - ROH within known disease gene regions warrants checking for homozygous pathogenic variants diff --git a/docs/13-vep-annotation.md b/docs/13-vep-annotation.md index c1e859f6e..b3c0ca43d 100644 --- a/docs/13-vep-annotation.md +++ b/docs/13-vep-annotation.md @@ -16,7 +16,7 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. ## Prerequisites - The VCF from step 3 **and its `.tbi`**: the step refuses a VCF without its index, which may be half written -- Offline VEP cache: step 13 downloads, checks and unpacks it the first time it runs (see step 00-reference-setup) +- Offline VEP cache: setup installs it by default. To retry, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"` ([reference setup](00-reference-setup.md#vep-cache-26-gb)). The pipeline skips VEP when the cache is missing. - Cache size: see [Hardware and storage requirements](hardware-requirements.md#shared-reference-data-one-time) (about 26 GB to download, 30 GB unpacked) ## Command @@ -112,4 +112,4 @@ After annotation, use step 23 (clinical filter) which automatically detects avai - `--dir_cache /opt/vep/.vep`: the cache is mounted there, not in the home directory VEP looks in by default - `--fasta`: running `--offline` without a FASTA file disables HGVS notation (`INFO: Disabling --hgvs`), so the script always passes the reference ([`REF_FASTA`](00-reference-setup.md#the-reference-path-on-every-page) is the reference path) - VEP does NOT assess variant pathogenicity in ClinVar context — combine with step 6 (ClinVar screen) for full picture -- **Upgrading from an older release:** VEP reads the cache directory named after the release passed in `--cache_version` (`homo_sapiens/116_GRCh38/` for the pinned release 116). When that directory is missing, VEP stops with an error instead of annotating from an older cache such as `112_GRCh38/`; step 13 downloads the matching cache on its first run. +- **Upgrading from an older release:** VEP reads the cache directory named after the release passed in `--cache_version` (`homo_sapiens/116_GRCh38/` for the pinned release 116). When that directory is missing, VEP stops with an error instead of annotating from an older cache such as `112_GRCh38/`. Setup installs the matching cache by default. The standalone step script can download a missing cache, but `run-all.sh` skips VEP until the cache is installed. diff --git a/docs/17-cpsr.md b/docs/17-cpsr.md index abf28e100..b652e9bbf 100644 --- a/docs/17-cpsr.md +++ b/docs/17-cpsr.md @@ -22,22 +22,21 @@ The image is PCGR 2.3.2 from `ghcr.io/sigven/pcgr` (Docker Hub's `sigven/pcgr` s ### 1. VEP Cache PCGR 2.3.2 bundles VEP 115, which requires the **release-115** cache (~24 GB). This is different from the release-116 cache used by step 13. Both coexist in the same `vep_cache/` directory under different subdirectories (`116_GRCh38/` and `115_GRCh38/`). -```bash -mkdir -p ${GENOME_DIR}/vep_cache -wget -c -P ${GENOME_DIR}/vep_cache https://ftp.ensembl.org/pub/release-115/variation/indexed_vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz -tar xzf ${GENOME_DIR}/vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz -C ${GENOME_DIR}/vep_cache -``` -### 2. PCGR Ref Data Bundle -PCGR 2.x uses a ref data bundle (~7 GB for `20260620`) separate from VEP: +### 2. PCGR ref data bundle +PCGR 2.x uses a separate ref data bundle (`20260620`, about 7 GB). +A plain setup installs the VEP 115 cache and this bundle by default, alongside VEP 116 for step 13. +For an existing setup, install or retry all three with: + ```bash -mkdir -p ${GENOME_DIR}/pcgr_data -cd ${GENOME_DIR}/pcgr_data -wget -c https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.20260620.grch38.tgz -tar xzf pcgr_ref_data.20260620.grch38.tgz -mkdir -p 20260620 && mv data/ 20260620/ +./scripts/setup.sh --clinical-data "$GENOME_DIR" ``` -This creates a `20260620/data/` directory with ClinVar, CancerMine, UniProt, and other databases. VEP cache is now mounted separately. + +The bundle goes to `pcgr_data/20260620/data/`; the caches coexist under `vep_cache/homo_sapiens/`. +Setup checks Ensembl's cache checksums and the bundle's pinned SHA-256 before extraction, then removes each archive. +It checks free space for each install and continues with a warning if one fails or does not fit. +`--no-clinical-data` skips all three. Without either CPSR item, `run-all.sh` skips step 17. +See [reference setup](00-reference-setup.md) for disk peaks and the [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) for the bundled databases. Review each source's terms before use. ### Upgrading from PCGR 2.2.5 PCGR 2.2.5 read bundle `20250314` and the release-113 cache. 2.3.2 needs both new ones: about 31 GB to download (the 7 GB bundle and the 24 GB cache). Once step 17 runs on 2.3.2, `pcgr_data/20250314/` and `vep_cache/homo_sapiens/113_GRCh38/` are no longer read and can be deleted. diff --git a/docs/20-mtoolbox.md b/docs/20-mtoolbox.md index 90951b026..4105c002f 100644 --- a/docs/20-mtoolbox.md +++ b/docs/20-mtoolbox.md @@ -119,6 +119,8 @@ NuMTs are copies of mitochondrial DNA in the nuclear genome. Their reads can map | 0.03-0.10 | Low-level heteroplasmy — may be age-related somatic | | <0.03 | Near detection limit | +Step 07 reads these calls for the aminoglycoside risk gene MT-RNR1: a PASS allele PharmCAT names (m.1555A>G, m.1494C>T and others) at AF 0.05 or more is passed to PharmCAT, heteroplasmic or not, because CPIC sets no heteroplasmy level at which aminoglycosides are safe ([Pharmacogenomics](07-pharmacogenomics.md#input-the-preprocessor-or-pharmcat-refuses)). + Count only PASS calls: a call marked `possible_numt` or with another filter is not evidence of heteroplasmy. Calls near the two ends of the control region (about chrM:1-500 and 16,000-16,569) are less reliable: the reference is circular, so reads that span its end align poorly at both edges. Length variants in the D-loop poly-C tracts (m.303-315 and m.16184-16193), one C more or less, are found in most people and often read as a mix of lengths. The reports list them as "length heteroplasmy in the D-loop poly-C tracts" and leave them out of the heteroplasmic count. A call counts as in a tract when the bases it changes (an indel: the bases after its anchor base) overlap one. diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index 101b425f5..28388bf82 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -51,8 +51,10 @@ Missense variants and in-frame insertions/deletions, on the ranked transcript. The same impacts with no frequency value. Their own tier, not rare (see above). ### ClinVar pathogenic/likely pathogenic, at any frequency -- Preferred source: the step 6 hits file, built from the ClinVar file in `clinvar/` that `setup.sh` refreshes. The tier holds the records at those positions. -- Step 6 matches on a split, left-aligned copy of the sample, and this tier selects the VEP records at the same CHROM and POS. An SNV always matches. An indel matches only when the caller already wrote it left-aligned, as DeepVariant does; an indel whose position moves on left-alignment is missing from this tier, though it stays in the step 6 hits and in both reports' ClinVar section. +- Preferred source: the step 6 hits file, built from the ClinVar file in `clinvar/` that `setup.sh` refreshes. The tier holds the VEP records whose allele is a hit: the same CHROM, POS, REF and ALT. +- Another allele at a hit's position is not in the tier. A call of T>A,C splits into two records at one position; when ClinVar classified only T>A, the T>C record stays out. +- Step 6 and step 13 both split and left-align the sample against the same reference, so one allele is written the same way in the hits and in the VEP output, one ALT per record. A VEP record with several ALTs (an input step 13 did not split) is kept when any of its ALTs is a hit. +- A combined hit of step 6 (two neighbouring records that make one allele, `PGP_JOINED_FROM` in the hits) puts both of its parts in the tier. VEP annotated the parts, not the joined allele. The tier's count then counts each part as one record. - Next, when step 6 has not run: `ClinVar_CLNSIG`, the same ClinVar file step 13 annotated with `--custom` (pathogenic or likely pathogenic, not conflicting). It follows a ClinVar refresh once step 13 runs again. - Last: VEP's `CLIN_SIG`, from the VEP cache release, so a ClinVar refresh never reaches it. The step says which source it used. - A common pathogenic allele (for example HFE p.C282Y) stays: this tier has no frequency filter. diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 9f1941391..650c9de24 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -15,6 +15,7 @@ Most blood group antigens differ by one known variant, so the genome predicts th ## Tool - `og bloodtype report` (OwnGenome's Rust binary, `rust/crates/og-bloodtype`), applying the marker table `assets/bloodtype_markers.tsv`. It reads the marker sites through the VCF's `.tbi` or `.csi`; without a usable index (none, or one older than the VCF) it reads the VCF once, line by line, with the same result. `og bloodtype check-markers` checks that the table and the rules agree +- bcftools `norm` to left-align the VCF's records near the marker sites against the reference first (see [Indels written another way](#indels-written-another-way)) - samtools `bedcov` for the read depth over the marker sites and RHD; the RHD regions are the RHD row of `bin/cn_loci.tsv`, and og cn's engine turns their depth into the copy number (see [Copy number from depth](cn-loci.md)) The variants and the antigens they define are the ISBT allele definitions, as RBCeq2 2.4.4 packages them (database v2.5.1, MIT licence, Australian Red Cross Lifeblood). The table's header names the release and the checksum of the file they were read from. Each position and REF base was checked against Ensembl GRCh38 and its dbSNP id. The HPA positions come from dbSNP through Ensembl, because that RBCeq2 database lists them at their GRCh37 positions. @@ -27,13 +28,14 @@ Why not run RBCeq2 or bloodTyper itself: RBCeq2 is on PyPI only, with no Biocond ## Docker Image -- `PYTHON_IMAGE` for the rules, `SAMTOOLS_IMAGE` for the depth +- `PYTHON_IMAGE` for the rules, `SAMTOOLS_IMAGE` for the depth, `BCFTOOLS_IMAGE` for the left-alignment. In the Nextflow pipeline the `BLOODTYPE` process runs both the left-alignment and the rules in `BCFTOOLS_IMAGE` (`og` is a static binary) Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, left-aligned records (DeepVariant writes them so). Split multi-allelic records, all-sites VCFs (a reference record and then the variant at one position) and gVCF reference blocks are read too: every record that covers a site counts, whatever their order (see below) +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, from any caller. Its records near the marker sites are left-aligned first, so an indel need not be written the way the table writes it. Split and whole multi-allelic records, all-sites VCFs (a reference record and then the variant at one position) and gVCF reference blocks are read too: every record that covers a site counts, whatever their order (see below) +- the reference (`REF_FASTA`, `reference/GRCh38_no_alt_analysis_set.fasta` by default) with its `.fai`, for the left-alignment; `--reference` in the Nextflow pipeline - `${SAMPLE}/aligned/${SAMPLE}_sorted.bam` with its `.bai` (`ALIGN_DIR` as for the other BAM steps). Optional: without it RhD is not determined, and a site the VCF has no record for is read as the reference allele at confidence medium. ## Command @@ -64,6 +66,19 @@ With `run-all.sh`: `TOOLS=...,bloodtype`. It is opt-in and needs no data, so a r A site the VCF has no record for is the reference allele when the BAM has at least 8 reads there at MAPQ 20, and not determined below that. Without the BAM it is read as reference, and the confidence says so. The step leaves out of `samtools bedcov` the rows on a contig the BAM has no `@SQ` line for (a BAM cut to a few chromosomes), because `bedcov` would stop on them. Those sites then have no depth row: the ones with no VCF record are not determined, and RhD is not determined without chr1. A record at a site whose REF is not the table's REF is a no call, because it says nothing about the table's ALT. There are two exceptions. A 0/0 call over a longer REF that starts with the table's REF is the reference allele. A 0/0 record over a shorter REF (an all-sites VCF's `C . 0/0`, or a gVCF block that starts at a deletion marker such as ABO c.1061delC) says nothing past its own base, so the site reads as one with no record: the reads decide. +### Indels written another way + +The table writes each indel left-aligned, the way `bcftools norm` writes it: ABO c.1061delC is `CG C` at chr9:133255669, at the start of a run of G. A caller can write the same deletion further right in the run, as `GG G` at chr9:133255671 (DeepVariant does so in rare cases without `--normalize_reads`, which the pipeline does not pass, and a VCF from another caller may do so too). Matched by position as written, that record would miss the marker, and with reads there the site would read as the reference allele at confidence high. So the step first takes the VCF's records within 100 bp of each marker site and left-aligns them against the reference (`bcftools norm -c w -f`). 100 bp is more than the 38 bp the longest marker indel can shift along its repeat on GRCh38. The rules then read that file. + +- A multi-allelic record stays one record (no `-m -any`). A `C A,G 1/2` at ABO c.803 keeps its one B copy, and a null row's `1/2` of two of its ALTs keeps both copies. +- A multi-allelic indel is left-aligned as a whole, so its REF can stay longer than the marker's (`TCC TC,T` against `TC T`). Its REF then differs from the table's, and the site is a no call, as the paragraph above says. +- A record whose REF differs from the reference is kept as it is (`-c w`), with a warning in the log. +- A record on a contig the reference lacks (a VCF named `1`, `2`, ... against the chr-named reference) cannot be aligned. It goes through unchanged. +- bcftools reads the VCF through its index when the index htslib loads (the `.csi` if there is one, else the `.tbi`) is not older than the VCF. Otherwise it reads the VCF once from the start. Read that way, a VCF whose header has no `##contig` line for a marker's contig first gets one, because bcftools writes no record on an undeclared contig. That path also reads a VCF in plain gzip, not bgzip. +- The left-aligned file is a work file: the bash step deletes it after the rules run, and the pipeline does not publish it. + +### Several records at one site + When several records cover one site, each one is read and their order does not matter: - A call of the table's ALT wins over the other records at the site's position. An all-sites VCF writes `T . 0/0` and then `T TC 0/1` at ABO c.261, so the insertion counts. Another allele that a second record calls there (a split `T G 1/0`) is named in the note. @@ -102,7 +117,7 @@ Lacking an antigen is common (most people lack HPA-1b, for example). It means a ## Runtime -Not measured on a whole genome. The step reads the marker positions through the VCF's index (the VCF once, line by line, without one), and samtools reads the BAM at 284 small regions. +Not measured on a whole genome. bcftools reads the VCF's records near the marker sites through its index (the VCF once from the start without one) and left-aligns them; og reads the marker positions of that small file; samtools reads the BAM at 284 small regions. ## Notes diff --git a/docs/42-traits.md b/docs/42-traits.md index 90a25c9cd..c4ce794c6 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -38,9 +38,9 @@ In the Nextflow pipeline (`traits` in `--tools`, part of the default list) the p |---|---| | `traits/${SAMPLE}_traits.tsv` | one row per table row, then the APOE line: `rsid`, `gene`, `chrom`, `pos`, `trait`, `genotype` (reference allele first), `status`, `reading`, `ancestry`, `note`, `pmids` | | `traits/${SAMPLE}_genotype_rules.tsv` | one line per rule: `rule`, `gene`, `trait`, `rsids`, `chrom`, `pos`, `genotypes` (one per site, reference allele first), `gq` (the GQ of each site's record, a reference block's own GQ for a homozygous reference site, `.` when the gVCF header has no FORMAT/GQ), `phase`, `result`, `status`, `reading`, `ancestry`, `note`, `pmids` | -| `traits/${SAMPLE}_traits_genotypes.tsv` | what bcftools read at the table's positions and the rule sites: chromosome, position, REF, ALT, the genotype as bases and its GQ | +| `traits/${SAMPLE}_traits_genotypes.tsv` | what bcftools read at the table's positions and the rule sites: chromosome, position, REF, ALT, the genotype as bases, its GQ and the record's FILTER | -`status` is `called`, `not_covered` (no genotype at the position, from no reads or a no-call: not read) or `other_allele` (an allele the table does not describe, or an indel starting at the position: not read). A record with a missing allele (a no-call or a half call such as `./T`) is dropped by the genotype filter, so its position reads as `not_covered`. A gVCF whose reference base differs from the table's GRCh38 base stops the step: the gVCF is on another build. +`status` is `called`, `not_covered` (no genotype at the position, from no reads or a no-call: not read), `low_gq` (a low-quality call: not read) or `other_allele` (an allele the table does not describe, or an indel starting at the position: not read). A call is low quality when its GQ is below 20, or when the caller filtered it (a FILTER other than `PASS`, `.` or DeepVariant's `RefCall`, such as `LowQual`). The row keeps its genotype, and the reading names the GQ or the filter. GQ 20 is read, and so is a call without a GQ (`.`, a gVCF with no FORMAT/GQ). A reference block counts the same way: a homozygous reference site in a block with GQ 5 reads `low_gq`, not reference. A rule with one such site reads `low_gq` with no result, unless another site has no genotype (`not_covered`). The floor of 20 is the one `og pgx refblock-fill` and the Y private SNPs use; a call of one or two reads gets a GQ near 0, so an APOE e4/e4 or an HFE C282Y/C282Y is never read from it. A record with a missing allele (a no-call or a half call such as `./T`) is dropped by the genotype filter, so its position reads as `not_covered`. A gVCF whose reference base differs from the table's GRCh38 base stops the step: the gVCF is on another build. Both reports have a Traits section: every row with its genotype, reading, note and PubMed id, and a mark on the rows whose reading depends on ancestry. The APOE lines come last, after a warning, and in the HTML report they are folded away until you open them. Both reports also show the genotype rules (`traits/${SAMPLE}_genotype_rules.tsv`) under the traits rows: each rule's sites, genotypes, result or status, how it was phased, and its reading. The APOE rule is left out there, because its line is already in the traits table and stays folded away. The three TAS2R38 rows of the traits table are each read alone, and their note points at the rule line. @@ -91,7 +91,7 @@ The two haplotypes a person's genotypes make are looked up by their alleles, and - **`needs_phase no`** (APOE): the combination line is the one listing every pair the genotypes allow. Heterozygous at both APOE sites allows ε2/ε4 and ε1/ε3; that line reads ε2/ε4 and says it could also be the rare ε1/ε3. - **`needs_phase yes`** (TAS2R38): with one heterozygous site or none, the two haplotypes follow from the genotypes. With two or more, WhatsHap phases them from the reads: a read, or the two reads of a pair, that covers two of the sites shows which alleles sit on one copy. When every heterozygous site of the rule lands in one phase block the rule is read (`phase` read_backed); otherwise it is `unphased`, not read. The three TAS2R38 sites lie within 741 bases, so a read pair can span them, but not always. -`status` of a rule line is `called`, `not_covered` (a site has no genotype), `other_allele` (an allele the rule does not describe), `other_haplotype` (a copy that is none of the rule's haplotypes; it names the alleles) or `unphased`. `phase` is `not_used` (needs_phase no), `not_needed`, `read_backed` or `unphased`. +`status` of a rule line is `called`, `not_covered` (a site has no genotype), `low_gq` (a site's call has a GQ below 20 or a caller filter: not read, no result), `other_allele` (an allele the rule does not describe), `other_haplotype` (a copy that is none of the rule's haplotypes; it names the alleles) or `unphased`. `phase` is `not_used` (needs_phase no), `not_needed`, `read_backed` or `unphased`. The rules today: @@ -108,7 +108,7 @@ The rules today: The six rules after TAS2R38 need no phase. Their sites lie close together on one copy (the two APOL1 G1 changes were in perfect linkage disequilibrium in Genovese et al. 2010; the 9p21 and 17q21.31 tags each mark one haplotype), or, for HFE, a double heterozygote reads as one change on each copy. A copy the rule does not describe, such as a 9p21 copy with only one of the two risk alleles, is `other_haplotype`, not read. What each rule leaves out, said in its note: -- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. +- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The G2 reading uses the same floor as the trait sites: a call over the deletion span with a GQ below 20, a G2 call whose FILTER is not `PASS` or `.`, or a reference call whose FILTER is not `PASS`, `.` or `RefCall`, leaves G2 not read, so two G2 copies at GQ 5 never read G2/G2. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. - SERPINA1: M is any copy with neither S nor Z; rarer deficiency and null alleles are not read. A serum level confirms a deficiency. - HFE: the C282Y homozygote is also a secondary finding (`assets/acmg_sf.tsv`, [step 23](23-clinical-filter.md)); the rule gives the reading of every genotype. - GC: the protein levels are those of Powe et al. (2013), whose monoclonal assay was questioned in later letters. A protein type, not a vitamin D deficiency. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index d9689a32c..eb2690bc0 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -19,14 +19,22 @@ IBDmix needs no unadmixed reference population. It scores each site from the mod ### One genome and the stand-ins -IBDmix takes the modern allele frequency of each site from the modern genomes in its input. With one genome the frequency is that person's own genotype, and IBDmix drops every site below its minimum allele count, so nothing scores. The step therefore adds 100 stand-in people whose genotypes carry the 1000 Genomes phase 3 allele count of the chosen population at each site (`ARCHAIC_POP`, default `ALL`; `AFR`, `AMR`, `EAS`, `EUR` or `SAS`). IBDmix scores each modern column from the site's frequency and that column's genotype only, so your column gets the score it would get inside a population sample with that frequency. The stand-ins' own rows are dropped. +IBDmix takes the modern allele frequency of each site from every modern column in its input, and your column is one of them. With one genome the frequency is that person's own genotype, and IBDmix drops every site at its minimum allele count (`-m 1` needs 2 ALT and 2 REF alleles, and one person has 2 alleles), so nothing scores. The step therefore adds 100 stand-in people. Together they carry round(AF × 200) ALT alleles at each site, where AF is the 1000 Genomes phase 3 frequency of the chosen population (`ARCHAIC_POP`, default `ALL`; `AFR`, `AMR`, `EAS`, `EUR` or `SAS`). The stand-ins' own segments are dropped. + +So the frequency IBDmix uses at a site is (the stand-ins' ALT count + yours) / 202, or / 200 where your genotype is unknown. That is you inside a sample of about 100 people, about the size of one 1000 Genomes group (they hold 61 to 113 people). Chen et al. ran IBDmix the same way: once per 1000 Genomes group, with the scored person counted in the group's frequency. What follows from it: + +- With `-m 1`, IBDmix keeps a site only when the modern columns hold at least 2 ALT and at least 2 REF alleles. Your own alleles count, so your genotype can add a site or remove one. +- An AF at or under 0.0025 gives the stand-ins 0 ALT alleles (the rounding is half to even, so 0.0025 gives 0). A heterozygous call scores nothing there, and a homozygous ALT call is kept only by your own two copies. +- An AF from just over 0.0025 to just under 0.0075 gives the stand-ins 1 ALT allele, so the site is kept whenever you carry the ALT. Each step of one allele covers an AF band about 0.005 wide: 0.0026 and 0.0074 score the same. +- An AF at or over 0.9975 gives the stand-ins 200 ALT alleles. A homozygous ALT call then removes the site, and a 0/0 call keeps it. +- Chen et al.'s group count is a random sample, and the stand-ins carry the rounded expected count. Near an AF of 0.002 a real group would keep the site some of the time, and step 43 never does. This is a small difference in one direction, and the stand-ins carry a frequency of the whole population or continent, not of one group (see the same-person comparison under Runtime). ### The calls - IBDmix with the options of Chen et al.: LOD 4 (`-d 4`), minor allele count 1 (`-m 1`), archaic error 0.01 (`-a`), modern error 0.002 (`-e`), `-c 2` - Segments with LOD 4 or more and 50 kb or longer, the cutoffs of the paper - **Neanderthal**: the union of the segments against Altai and against Vindija. **Denisovan**: the segments against the Denisovan that overlap no Neanderthal segment, because the Neanderthal and Denisovan genomes share much of their sequence. -- The share is the segment length over the analysed span: from the first to the last site of the table on each autosome the VCF has calls on. +- The share is the segment length over the analysed span: from the first to the last site read on each autosome the VCF has calls on. A site is read when the VCF has a usable call there, or when a confident reference block of the gVCF covers it (see Input). ## Docker Image @@ -48,6 +56,8 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. | 1000 Genomes | the GRCh38 strict mask, `20160622.allChr.mask.bed` (128 MB) | sha256 | | UCSC | `hg19ToHg38.over.chain.gz` | sha256 | +> **Terms:** MPI-EVA makes the snpAD genotypes and the FilterBed masks freely available under the terms in https://cdna.eva.mpg.de/neandertal/Vindija/README: users are asked to observe the Fort Lauderdale principles, and the data can be used freely for studies of individual genes or other features of the genome. All three genomes' files come from that folder. Cite the genome papers in References. MPI-EVA gives no standard licence such as an SPDX one. The pipeline does not redistribute these files: `setup.sh --archaic` downloads them from MPI-EVA onto your disk, and deletes each chromosome's files once its table rows are built. + MPI-EVA publishes md5 lists, not sha256 sums, for 66 genotype files of about 2.5 GB each. Pinning the sha256 of the three lists pins every file's md5 without downloading 150 GB to hash it. The FilterBed files have no checksum file beside them and are about 1.5 MB each, so each one's sha256 is pinned directly. ### The masks @@ -69,14 +79,17 @@ The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf` ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A site with no record is read as the reference allele, a record whose FILTER is not PASS as unknown, and a site where you carry another ALT than the table's is left out. +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A record whose FILTER is not PASS is read as unknown, and a site where you carry another ALT than the table's is left out. A reference call with ALT `.`, as an all-sites VCF writes it, is the reference when its GT is all 0, and unknown otherwise. +- `${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz`: the gVCF of step 3, one sample. A variant-only VCF cannot tell a site that matches the reference from a site without reads, so a table site with no record in the VCF is read as the reference only where a confident reference block of the gVCF covers it. A block is confident when its FILTER is PASS or `.`, every allele of its GT is 0, and its GQ and its depth (MIN_DP, else DP) are 1 or more where it gives them. Any other site with no record goes to IBDmix as unknown (9), which IBDmix skips. Without the gVCF the bash step stops and says why, and the Nextflow pipeline skips the sample with a warning (add the `gvcf` and `gvcf_index` columns to its samplesheet row, or start from the BAM or the reads, as for steps 42 and 44). The gVCF must list the chromosomes in the VCF's order, with records on every table chromosome the VCF lists. - The GENCODE gene lines `setup.sh` installs, for the genes of each segment (without them the segments are listed without genes) -`og archaic genotypes` reads the VCF and table together, keeping one -duplicate-position window. Positions within each chromosome must be in -ascending order in both inputs. A position that goes backwards stops the -command with exit 2. Mask runs are written as they close. Duplicate calls, -clashes, masked sites and the output columns keep their previous meaning. +`og archaic genotypes` reads the VCF, the gVCF (`--gvcf`) and the table +together, keeping one duplicate-position window of the VCF and one record of +the gVCF. Positions within each chromosome must be in ascending order in all +three inputs. A position that goes backwards (the gVCF is read to its end +for this), a gVCF in another chromosome order, or a gVCF with no records on +a table chromosome of the VCF, stops the command with exit 2. Mask runs are written as +they close. It prints how many table sites it read and how many it did not. ## Command @@ -85,7 +98,7 @@ clashes, masked sites and the output columns keep their previous meaning. ./scripts/43-archaic.sh your_name ``` -With `run-all.sh`: step 43 runs once the table is installed (`ARCHAIC_DIR=none` skips it, `ARCHAIC_POP` picks the frequencies). In the Nextflow pipeline: `archaic` in `--tools` with `--archaic_data ` and `--archaic_pop`; `--hla_genes` gives the genes (processes `ARCHAIC_GENOTYPES`, `ARCHAIC_IBDMIX`, `ARCHAIC_REPORT`). +With `run-all.sh`: step 43 runs once the table is installed (`ARCHAIC_DIR=none` skips it, `ARCHAIC_POP` picks the frequencies). It needs the gVCF: a sample that starts from a VCF is listed as `skipped (no gVCF: ...)`, as for step 42, unless it has a BAM too and a gVCF with its index beside the VCF (the gVCF `run-all.sh` passes on). In the Nextflow pipeline: `archaic` in `--tools` with `--archaic_data ` and `--archaic_pop`; `--hla_genes` gives the genes (processes `ARCHAIC_GENOTYPES`, `ARCHAIC_IBDMIX`, `ARCHAIC_REPORT`). ## Output @@ -105,6 +118,8 @@ Measured on public HG002 by the `archaic-hg002` jobs of `public-evidence.yml`, o - Install: the archaic files were 150.5 GB in all, from 1.8 GB (chromosome 22) to 13.8 GB (chromosome 2). Downloading them took 2 h 32 min summed over the 22 chromosomes, from 3 to 32 minutes each, at the speed MPI-EVA served that day. Building the table took 3 h 15 min summed, from 2 to 20 minutes per chromosome. The finished table is 0.44 GB with 51,812,908 sites. - Step 43, one population: 19 to 88 seconds per chromosome, 18 minutes summed over the 22. IBDmix runs once per chromosome and archaic genome, `THREADS` chromosomes at a time. +Every figure in this section was measured before step 43 read the gVCF, when each site GIAB's variant-only VCF had no call for was read as the reference, inside GIAB's confident regions or not. The e2e case now gives GIAB's confident regions as the reference blocks and prints its chr22 figure on every run. The `archaic-hg002` jobs of `public-evidence.yml` give the same blocks, one per GIAB benchmark interval, so the next release run measures the whole-genome figure on this rule. The `archaic-same-person` job has no callable mask for the 1000 Genomes panel data, so it gives one reference block over each whole chromosome: a site where the panel has no record for the person reads as the reference, as it did when the table below was measured, and a site where the panel has no call for the person stays unknown. + That run was made before the masks, from a table that held every lifted site. It gave HG002 115.2 Mb of Neanderthal sequence over the 22 autosomes, about twice what Chen et al. report for Europeans (a median of 51.6 Mb per person, and 42.0 to 62.0 Mb across the released per-person calls of their 503 Europeans; see the same-person comparison below for what their method is). The cause was the missing masks: the table carried the positions each archaic genome's own filters exclude, and IBDmix scored them. On chr22, 43% of that run's Neanderthal sequence lay outside the 1000 Genomes GRCh38 strict mask alone. With the masks dropping sites, a run on chr22 (run 42089578470, before the masked codes) gave HG002 1.3 Mb of Neanderthal sequence with the `ALL` frequencies and 1.7 Mb with `EUR`, still at the top of the published spread below. Keeping the masked sites as IBDmix's mask, as the published workflow does, lets the mismatches there end a segment; the chr22 figure on this table is printed by the e2e case on every run. The e2e case compares it with the spread of Chen et al.'s own released calls for the 497 Europeans of CEU, TSI, GBR, FIN and IBS (51 kb at the lowest, 541 kb at the median, 1,132 kb at the 95th percentile, 1,716 kb at the highest). The pre-mask run gave 1,748 kb on chr22, more than all 497 of them. With the masked codes but the strict-mask rows still dropped, the fork runs gave 1,177 to 1,190 kb. Keeping those rows marked, the last difference from the published workflow in how masked sites are handled, gave 1,138 kb on chr22 (run 42335592918). One chromosome of one person varies widely, so the case gates on a regression guard of 1,200 kb (and on the lowest published value, 51 kb), and prints the published figures beside the measured one. It does not claim the figure is inside the published range. The whole-genome total is measured by the release run of `public-evidence.yml`, which installs the 150 GB of archaic data once per release, not by the e2e case, and printed beside the published 42.0 to 62.0 Mb; it is not gated on it, for the reasons in the same-person comparison below. The case prints where HG002 falls inside the spread; it does not gate on the median or the 95th percentile, because one person's chromosome is one draw from that distribution. The same-person comparison below shows the input's coverage is not why the step finds more than the published calls do. @@ -132,10 +147,10 @@ What this means for the figure step 43 reports. It is the union of the Altai and - Part of what IBDmix finds in anyone is old ancestry shared with the archaic genomes, not gene flow from them. Chen et al. find Neanderthal sequence in every population they studied, Africans included. - A segment counts when either of your two copies carries it. The share is of the analysed span, not of the whole genome. - The table has autosomes only: chrX and chrY are not assessed. -- Sites the VCF has no record for are read as the reference allele. A VCF that holds only variants (no reference calls) cannot tell a reference call from a gap in coverage. -- The share is of the analysed span, and the span still runs from the first to the last table site of each chromosome whatever the masks remove. A segment runs from its first site to the site after its last, so it can cross masked sequence and count those bases (a homozygous mismatch at a masked site still ends it, as in the published workflow); a wrong genotype at a site that was kept can raise the share as well. The number is an estimate in both directions, not a floor. Compare it with the published per-person figures, which come from masked data too. +- A site the VCF has no record for is read as the reference allele only inside a confident reference block of the gVCF. Elsewhere it is unknown, so a gap in coverage cannot make a segment, and a missed homozygous ALT call cannot end one as a false mismatch. Before 1.0.0 every such site was read as the reference: one call anywhere on a chromosome was enough to analyse all of it, and a stretch with no reads where the archaic genomes carry the common reference allele could score as a long segment. +- The share is of the analysed span, and the span runs from the first to the last site read on each chromosome whatever the masks remove. A segment runs from its first site to the site after its last, so it can cross masked sequence and count those bases (a homozygous mismatch at a masked site still ends it, as in the published workflow); a wrong genotype at a site that was kept can raise the share as well. The number is an estimate in both directions, not a floor. Compare it with the published per-person figures, which come from masked data too. - The CpG sites Chen et al. exclude are kept here. A C to T change at a CpG site is the commonest ancient-DNA error, so a few of the archaic genotypes at those sites are wrong in both directions, and the sites are over-represented among the matches a method like this finds. Their own filter needs data this pipeline does not install, so the share carries that error; it is one of the reasons a number from this step is an estimate. -- On the e2e fixture the case builds the table for GRCh37 chr22 and checks HG002's GIAB v4.2.1 genotypes against the published chr22 spread, a copy of HG002 with one chromosome over chr22:28-29 Mb taken from Vindija (a Neanderthal segment must cover it), a person built from the table who carries only the majority allele (a quarter of HG002's Neanderthal sequence at most), that every table row outside the strict mask carries only masked calls, and that the FilterBeds and the exclusions marked calls masked. +- What the tests check. The unit tests check the table build, the masks, the stand-in counts, the gVCF rule and the report's merging on synthetic data. With the pinned IBDmix image they also check a planted Neanderthal and Denisovan stretch, a masked mismatch that must end a segment, and a stretch with no call that must give no segment. They do not check that the share is calibrated against published per-person totals (by design it is not; see Runtime), and the stand-in rounding is pinned, not compared with a sampled population. On the e2e fixture the case builds the table for GRCh37 chr22 and checks HG002's GIAB v4.2.1 genotypes, with GIAB's confident regions as the gVCF's reference blocks, against the published chr22 spread, a copy of HG002 with one chromosome over chr22:28-29 Mb taken from Vindija (a Neanderthal segment must cover it), a person built from the table who carries only the majority allele (a quarter of HG002's Neanderthal sequence at most), that every table row outside the strict mask carries only masked calls, and that the FilterBeds and the exclusions marked calls masked. ## References diff --git a/docs/alerts.md b/docs/alerts.md index d2dac00fe..7f242948a 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -97,4 +97,4 @@ The hours left, the late steps and the unknown steps are written only while a ru ## How it is tested -`tests/test_owngenome_live_ops.py` runs the plugin on the made-up run of `tests/fixtures/live/ops/` with a fake disk and a fake host, then against planted mistakes that must fail it. The `promtool` job in `.github/workflows/lint.yml` runs `pgp-alerts.test.yml`, then changes one threshold and requires the test to fail. +`rust/crates/og-live/tests/ops.rs` runs og-live's ops module, the Rust port of the plugin, on the made-up run of `tests/fixtures/live/ops/` with a fake disk and a fake host. It also runs the plugin itself on the same inputs: the card must match by meaning and the metrics text byte for byte. `ops_eta.rs` and `ops_headroom.rs` beside it check the hours left and the headroom. Each planted mistake of the `fault-inject` feature must fail its check: Guard's rust job runs them and requires every one to fail. The `promtool` job in `.github/workflows/lint.yml` runs `pgp-alerts.test.yml`, then changes one threshold and requires the test to fail. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 5d0eb63a7..2ef089c1b 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -19,18 +19,20 @@ For background on caller performance differences, see: ## Available Alternative Caller Scripts -The pipeline ships with four variant callers. Each writes output to a separate directory so they can run in parallel without conflicts. +The pipeline calls with DeepVariant by default and ships five alternative caller scripts: four for short reads and Clair3 for long reads. Each writes output to a separate directory so they can run in parallel without conflicts. The image each script runs is on [Image versions](versions.md). | Script | Caller | Output Directory | Description | |---|---|---|---| | `scripts/03-deepvariant.sh` | DeepVariant (`DEEPVARIANT_IMAGE`) | `vcf/` | Default caller. Deep learning model, highest precision and F1. | -| `scripts/03a-gatk-haplotypecaller.sh` | GATK HaplotypeCaller (`GATK_IMAGE`) | `vcf_gatk/` | Gold standard in clinical labs. Good precision/recall balance, GVCF support. | +| `scripts/03a-gatk-haplotypecaller.sh` | GATK HaplotypeCaller (`GATK_IMAGE`) | `vcf_gatk/` | Widely used in clinical labs, but this script is not their full GATK Best Practices workflow: it calls the duplicate-marked step-02 BAM without base quality score recalibration (BQSR) and writes a plain VCF (`-ERC NONE`), not a GVCF. Good precision/recall balance. | | `scripts/03b-freebayes.sh` | FreeBayes (image set in the script) | `vcf_freebayes/` | Bayesian caller. Highest sensitivity, most false positives, single-threaded. | | `scripts/03c-strelka2-germline.sh` | Strelka2 (image set in the script) | `vcf_strelka2/` | Fast heuristic caller (SNVs + indels). Best with BWA-MEM2 alignments. | +| `scripts/03d-octopus.sh` | Octopus (`OCTOPUS_IMAGE`) | `vcf_octopus/` | Haplotype-based caller. | +| `scripts/03e-clair3.sh` | Clair3 (`CLAIR3_IMAGE`) | `vcf_clair3/` | Long-read caller (ONT, PacBio HiFi) for a BAM from `scripts/02b-alignment-longread.sh`. | -All four scripts accept the same arguments: +DeepVariant, GATK, FreeBayes and Strelka2 take the same arguments: ```bash export GENOME_DIR=/path/to/your/data @@ -59,6 +61,19 @@ There are two ways to evaluate caller performance: Compare two or more caller VCFs against each other to measure agreement. This tells you how much the callers agree, but not which one is "right." +`./scripts/benchmark-variants.sh ` does this for every caller VCF it finds: `vcf/` (DeepVariant), `vcf_strelka2/results/variants/variants.vcf.gz`, and `.vcf.gz` in every `vcf_*/` directory (`vcf_gatk/`, `vcf_freebayes/`, `vcf_octopus/`, `vcf_clair3/`, or a directory of your own, named in the results by what follows `vcf_`). That is the same set that makes `run-all.sh` with `BENCHMARK=true` queue the step. It normalises each VCF once (`bcftools norm -m-both`), runs `bcftools isec` on PASS and unfiltered records for each pair, and writes `benchmark/comparison.tsv`: + +| Column | Meaning | +|---|---| +| `Shared`, `A_Unique`, `B_Unique` | sites both callers have, and sites only one has. A site is shared when both have the same allele at the same position, whatever the genotype | +| `Jaccard` | `Shared / (Shared + A_Unique + B_Unique)`: agreement on which alleles are present | +| `GT_Concordant` | shared sites where the two genotypes are also the same | +| `GT_Concordance` | `GT_Concordant / Shared`; `N/A` when nothing is shared | + +A site one caller calls 0/1 and the other 1/1 counts in `Shared` and in the Jaccard index, but not in `GT_Concordant`. Genotypes are compared without phase and with their alleles sorted (`1|0` equals `0/1`), and a homozygous call equals its haploid form (`1/1` equals `1`), so a caller that writes male chrX haploid still agrees with one that writes it diploid. A site with no genotype on either side (`./.`, `.`, or a record with no `GT` field) does not count as concordant. + +The commands below do the allele part by hand: + Use `bcftools isec` to compute the intersection and unique calls: ```bash @@ -252,7 +267,8 @@ Each hap.py run produces several output files: | **Precision** | How many called variants are real (low = too many false positives) | TP / (TP + FP) | | **Recall** | How many real variants were found (low = missing real variants) | TP / (TP + FN) | | **F1** | Harmonic mean of precision and recall (balanced accuracy) | 2 * (Precision * Recall) / (Precision + Recall) | -| **Jaccard** | Overlap between two call sets (pairwise mode) | Shared / (Shared + Unique_A + Unique_B) | +| **Jaccard** | Overlap between two call sets (pairwise mode): the same allele at the same position, whatever the genotype | Shared / (Shared + Unique_A + Unique_B) | +| **GT concordance** | Shared sites where the two callers also give the same genotype (pairwise mode) | GT_Concordant / Shared | Where: - **TP** (True Positive) = variant called correctly @@ -267,6 +283,7 @@ Where: - **Jaccard > 0.95**: Two callers agree on the vast majority of calls. - **Jaccard 0.85-0.95**: Moderate agreement. Differences are mostly in low-confidence regions or indels. - **Jaccard < 0.85**: Significant disagreement. Worth investigating the unique calls from each caller. +- **GT concordance well below 1 with a high Jaccard**: the callers find the same variants but disagree on zygosity. Look at the depth and allele balance at those sites. ### Typical Results (30X WGS, GRCh38) diff --git a/docs/getting-started.md b/docs/getting-started.md index 6067bc415..e72239787 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,7 +18,7 @@ | **CPU** | 4 cores | 8-16 cores | `run-all.sh` runs DeepVariant with at most 8 shards, fewer when `--max_cpus` is lower; more cores run other steps beside it ([CPU](hardware-requirements.md#cpu-requirements)) | | **RAM** | 16 GB | 32 GB | Every container has a hard memory limit; the 8-CPU steps ask for 32 GB, capped at the machine's RAM | | **Disk** | 500 GB free from a BAM, about 700 GB from FASTQ | 1 TB+ | See [detailed breakdown](hardware-requirements.md#disk-space-breakdown) | -| **Internet** | Broadband | 100+ Mbps | ~70-75 GB core downloads + ~175 GB optional annotation databases | +| **Internet** | Broadband | 100+ Mbps | ~73-78 GB core downloads + ~175 GB optional annotation databases | | **OS** | Linux (amd64) | Ubuntu 22.04+ | macOS/ARM works but slower (see below) | > **Disk space is the #1 surprise.** A single 30X WGS sample produces 60-90 GB of FASTQ, 80-120 GB of BAM, plus reference genomes and databases. See [docs/hardware-requirements.md](hardware-requirements.md) for the full breakdown. @@ -46,11 +46,17 @@ Every analysis tool runs inside Docker: no conda environments, no Python version | PCGR/CPSR data bundle + VEP 115 cache | ~31 GB | Step 17 (cancer predisposition) | | Docker images (all steps) | ~10-15 GB | All steps | | Annotation databases (CADD, SpliceAI, REVEL, AlphaMissense) | ~175 GB | Steps 30-31 (optional) | -| **Total one-time setup (core)** | **~70-75 GB** | | +| **Total one-time setup (core)** | **~73-78 GB** | | | **Total with annotation enrichment** | **~250 GB** | | See [docs/00-reference-setup.md](00-reference-setup.md) for download instructions. +Plain setup includes VEP 116, the PCGR/CPSR bundle and VEP 115: about 57 GB of clinical downloads and 65 GB unpacked. +Allow 1 to 3 hours. Setup checks each install's disk peak plus a 5 GB margin and warns if it skips one. +Use `./scripts/setup.sh --no-clinical-data "$GENOME_DIR"` to opt out. +Use `./scripts/setup.sh --clinical-data "$GENOME_DIR"` to add or retry only these data later. +`run-all.sh` skips steps whose data are still missing; inspect its plan and setup warnings. + ## Platform Notes ### Linux (Recommended) @@ -120,7 +126,7 @@ Then, once per machine and once per sample: ```bash export GENOME_DIR=/path/to/your/data -./scripts/setup.sh $GENOME_DIR # reference, ClinVar, small data files, images and og (once) +./scripts/setup.sh $GENOME_DIR # reference, ClinVar, VEP caches, CPSR bundle, images and og (once) ./scripts/validate-setup.sh your_name # what is installed and what is missing ./scripts/run-all.sh your_name male # or female: sets the chrX/chrY ploidy and the sex check ``` diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 67a7b09ee..1da8b16a0 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -43,7 +43,7 @@ A run from FASTQ needs more space than a run from a BAM. Until you delete ` [male|female]`. The other scripts set a fixed `--memory`; edit it in the script to change it. Step 25's pgsc_calc is the exception that does not fit: its first task needs about 14 to 18 GB on the default scores and was killed under a 14.1 GB limit when the host has no swap, so a 16 GB machine needs swap for it ([step 25, Memory](25-prs.md#memory)). +**Minimum system RAM:** 16 GB. `run-all.sh` passes the machine's RAM as `--max_memory` (the container engine's when it has less, as on Docker Desktop and colima), so on a 16 GB machine the 32 GB tasks run capped at about 15 GB. A task that needs more is killed (exit 137); its one retry asks for double, and the cap cuts that back to the same 15 GB. With the step scripts, lower DeepVariant's shards and memory with `THREADS` and `DV_MEM`, for example `THREADS=4 DV_MEM=12g ./scripts/03-deepvariant.sh [male|female]`. The other scripts set a fixed `--memory`; edit it in the script to change it. Step 25's pgsc_calc is the exception that does not fit: its first task needs about 14 to 18 GB on the default scores and was killed under a 14.1 GB limit when the host has no swap, so a 16 GB machine needs swap for it ([step 25, Memory](25-prs.md#memory)). **Recommended:** 32 GB or more, so minimap2 and DeepVariant get close to the 32 GB they ask for. **GRIDSS (step 4b, opt-in):** its container takes 32 GB (a 28 GB Java heap plus overhead), so it needs a machine with 32 GB or more even when nothing else runs. @@ -141,7 +141,7 @@ These are limits, not measured peaks. The two step 44 rows were measured on a 4- ## CPU Requirements -Each pipeline task asks for the CPUs that `conf/base.config` gives its label or its name: 1, 2, 4 or 8. The 8-CPU tasks are minimap2 (the index, and the alignment with its duplicate marking in the same task), fastp, DeepVariant, Manta and VEP. `--max_cpus` and `--max_memory` cap each task; `run-all.sh` passes the machine's CPU count (or `THREADS`) and its RAM. They do not limit how much runs at once: Nextflow starts tasks until their requests fill the machine's CPUs and RAM. On a Mac with Docker Desktop, that is the Mac's CPUs and RAM, not the Docker VM's. `MAX_JOBS` is no longer read. +Each pipeline task asks for the CPUs that `conf/base.config` gives its label or its name: 1, 2, 4 or 8. The 8-CPU tasks are minimap2 (the index, and the alignment with its duplicate marking in the same task), fastp, DeepVariant, Manta and VEP. `--max_cpus` and `--max_memory` cap each task; `run-all.sh` passes the machine's CPU count (or `THREADS`) and its RAM. They do not limit how much runs at once: Nextflow starts tasks until their requests fill the machine's CPUs and RAM. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac. When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` passes the VM's figures instead, in whole GB. It also writes them to `/nextflow/executor.config` and passes that file with `-c`. Nextflow then fills the VM, not the Mac, and a task that does not fit waits for room instead of being killed inside the VM. On native Linux the engine reports the host's figures, and nothing changes. `MAX_JOBS` is no longer read. **DeepVariant's shards.** `run-all.sh` runs DeepVariant with at most 8 shards, however many cores the machine has, and fewer when `--max_cpus` (`THREADS`) is lower. More cores help the steps that run beside it. To give DeepVariant more, run `scripts/03-deepvariant.sh` with `THREADS=N` (its `--cpus` and `--num_shards`), or pass `-c` with a `withName: 'DEEPVARIANT'` block: @@ -178,7 +178,7 @@ executor { } ``` -Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. The same `executor` block is the way to keep Docker Desktop's VM from being overbooked: set it to the CPUs and memory the VM has, and pass `THREADS` and `--max_memory` no larger than the VM's (e.g. `THREADS=6 ./scripts/run-all.sh --max_memory 14.GB`). On a Mac, `run-all.sh` otherwise passes the Mac's own CPU count and RAM, and Nextflow refuses a task that asks for more than the `executor` block allows. +Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. On Docker Desktop and colima, `run-all.sh` already writes an `executor` block with the VM's CPUs and memory to `/nextflow/executor.config`. It passes that file before your options, so each setting in an `executor` block of your own (`-c`) wins over it; a setting you leave out keeps the VM's figure. Keep `THREADS` and `--max_memory` no larger than your block, or Nextflow refuses the tasks that ask for more. Without a block of your own, `run-all.sh` never books less than a cap you set: `THREADS`, `--max_cpus` or a whole-GB `--max_memory` above the VM's figure raises the CPUs or memory in `executor.config` to it, and a `--max_memory` in other units leaves the memory to Nextflow. The run then starts as it would without the VM's figure, and a task the VM cannot hold can be killed (exit 137); `run-all.sh` prints a `[Resources]` line that says so. ### Runtime per step diff --git a/docs/interpreting-results.md b/docs/interpreting-results.md index 576e963c2..b236ba59a 100644 --- a/docs/interpreting-results.md +++ b/docs/interpreting-results.md @@ -28,669 +28,53 @@ If this is your first time looking at your own genomic data, the numbers can be | Heteroplasmic mitochondrial variants | 20-40 | Low-level heteroplasmy (<5%) is universal and age-related | | VUS (Variants of Uncertain Significance) | 20-200+ | "Uncertain" means **not enough data yet**, not "probably bad" | -### The VUS Trap - -The single biggest source of unnecessary anxiety in personal genomics is **VUS — Variants of Uncertain Significance**. These are variants where: - -- There is not enough scientific evidence to classify them as either pathogenic or benign -- The vast majority will eventually be reclassified as **benign** as more data accumulates -- They are **not actionable** — no clinical decision should be made based on a VUS -- CPSR may report dozens or hundreds of VUS. This is normal and expected. - -**Rule of thumb:** If a variant is classified as VUS, treat it the same as if it were not tested. Do not change screening or management based on a VUS. Check back in 1-2 years when ClinVar may have reclassified it. - -**One nuance:** If you have a strong family history of a condition AND a VUS appears in the relevant high-penetrance gene (e.g., BRCA1/2, TP53, MLH1/MSH2), it may be worth mentioning to a genetic counselor — not to act on the VUS, but because the family history itself may warrant enhanced screening regardless of the variant's classification. - -### ClinVar Star Ratings - -Not all ClinVar classifications are equally reliable. Each entry has a **review status** indicated by stars: - -| Stars | Review Status | Reliability | -|---|---|---| -| 0 | No assertion criteria | Low — submitter did not explain their reasoning | -| 1 | Single submitter, criteria provided | Moderate — one lab's interpretation | -| 2 | Two or more submitters, no conflict | Good — multiple labs agree | -| 3 | Expert panel reviewed | High — reviewed by specialists | -| 4 | Practice guideline | Highest — established clinical standard | - -**Focus on 2+ star entries.** Single-submitter (1-star) pathogenic calls are sometimes reclassified. If you find a scary-looking pathogenic variant with 0-1 stars, check the ClinVar entry directly at [ncbi.nlm.nih.gov/clinvar](https://www.ncbi.nlm.nih.gov/clinvar/) — look at the "Review status" and "Last evaluated" date. - -### Carrier Status Is Not Disease - -The most common "pathogenic" finding in any genome is **heterozygous carrier status for recessive conditions**. This means: - -- You have ONE copy of a variant that causes disease when BOTH copies are affected -- For most recessive conditions, heterozygous carriers are **not clinically affected** (though some carrier states confer subtle phenotypic effects — e.g., sickle cell trait, HFE carriers and iron loading) -- The primary relevance is for **family planning**: if your partner carries the same gene, each child has a 25% chance of being affected -- **Note — MUTYH**: Biallelic (homozygous or compound het) MUTYH carriers have a well-established high colorectal cancer risk. For **monoallelic** (single-copy) carriers, the evidence is more nuanced: some meta-analyses show a modest risk increase, but a counseling framework for moderate-penetrance CRC genes (Genetics in Medicine) notes that risk estimates for monoallelic MUTYH are conflicting and that screening recommendations (e.g., earlier colonoscopy) were historically tied to carriers with a CRC family history. Discuss with a genetic counselor, especially if you have a family history of CRC -- Examples: GJB2 (hearing loss), CFTR (cystic fibrosis), HFE (hemochromatosis) - -**Your own carrier list (step 24 report).** The ClinVar card of both reports is followed by a carrier list for this one sample, with no partner needed: every step 6 ClinVar hit in the 3,292 recessive and X-linked genes of `assets/carrier_genes.tsv` (the list of the [couple screen](39-couple.md#the-carrier-list)). An allele that list marks as a hypomorph (for example BTD p.Asp444His or the GALT Duarte variants) or a pseudodeficiency allele (HEXA c.739C>T and c.745C>T, which lower the enzyme test result but cause no Tay-Sachs disease) is listed as set aside, never as a carrier finding on its own. - -**Secondary findings follow each gene's ACMG rule.** The secondary-findings card (step 23's section of the report) applies the reporting rule of each of the 84 ACMG SF v3.3 genes (Lee et al., Genet Med 2025;27(8):101454, Table 1), written gene by gene in `assets/acmg_sf.tsv`: - -- a dominant gene (BRCA1, LDLR, MYH7 and most others): one pathogenic or likely pathogenic variant is reported; -- a recessive gene (MUTYH, ATP7B, BTD, CYP27A1, GAA, RPE65, CASQ2, TRDN): two variants are needed, a homozygote or two different ones even when their phase is unknown. One heterozygous MUTYH variant is listed apart as not reported, with the reason; -- HFE: only homozygous p.Cys282Tyr. A heterozygous p.Cys282Tyr or a homozygous p.His63Asp is not reported; -- X-linked: GLA (Fabry disease) and OTC are reported in a hemizygous male and a heterozygous female alike. ABCD1 (X-linked adrenoleukodystrophy) is reported hemizygous, homozygous or with two heterozygous variants, never for one heterozygous variant; -- BTD with p.Asp444His: reported as partial, not profound, biotinidase deficiency; -- AnnotSV class 4 and 5 structural variants (step 5) in BRCA1, BRCA2, MLH1, MSH2 and LDLR reach the same card, and so does a deletion of the 3' end of EPCAM (exons 8 and 9), which silences MSH2 (Kovacs et al., Hum Mutat 2009;30(2):197-203). - -**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`og summary collect`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: - -- **biallelic** when the reads show the two hits on different copies. Either the hit is called homozygous inside the deletion, so it sits on the one copy left (the line and the carrier list call it hemizygous), or both calls carry read-backed phase (a `|` genotype with the same phase set, PS) on different copies; -- **possibly biallelic** otherwise: the phase is unknown. Most short-read calls carry no phase, so this is the usual line. Step 05's checked table keeps no genotype column yet, and step 35's counts have no phase, so in a pipeline run a deletion pairs as possibly biallelic unless the hit is called homozygous inside it. A parent's genome (step 38, [family analysis](multi-sample.md#family-analysis-step-38-opt-in)) or long reads can settle it; -- no line when read-backed phase puts both on the same copy (cis): the other copy is intact. - -Examples: an exon deletion of MUTYH plus a pathogenic MUTYH variant; three alpha-globin genes plus an HBA2 point variant; one SMN1 copy plus an SMN1 variant (short reads place few SMN1 variants reliably, so check with a targeted test); and the 1q21.1 deletion plus one of the two RBM8A low-expression alleles (c.-21G>A, rs139428292, or c.67+32G>C, rs201779890), the cause of TAR syndrome (thrombocytopenia with absent radius; Albers et al., Nat Genet 2012;44(4):435-439, PMID 22366785). On its own an RBM8A low-expression allele stays set aside. The [couple screen](39-couple.md) (step 39) applies the same pairing between partners: a hypomorph allele of the list in one partner and a full loss of function in the same gene in the other (a ClinVar pathogenic variant, a Delly deletion) is a shared gene with its risk for a child; the allele in both partners, or in one alone, stays set aside. In the secondary-findings card a called deletion counts as one of the two variants a recessive gene needs, unless read-backed phase puts it on the variant's copy. The line under the carrier list names the deletion callers it read and the ones it could not read. A deletion AnnotSV ranks below class 4 is not counted. - -The TTN rule (truncating variants only) and the LMNA rule (missense only with cardiac evidence) are printed beside the finding, not applied. It is still a list to review with a clinician. - --- ## Start Here: The Three Most Important Outputs -### 1. ClinVar Screen (Step 6) - -**What it tells you:** Known pathogenic variants in your genome, as classified by ClinVar (NCBI's public database of clinically significant variants). - -**Where to look:** `${SAMPLE}/clinvar/` - -**How to read it:** -- Each line in the output VCF is a variant in your genome that matches a known ClinVar entry -- Step 6 intersects against the **pathogenic-only** ClinVar subset (Pathogenic + Likely_pathogenic). Every hit in this output is at a position ClinVar classifies as disease-associated — benign/VUS entries are excluded at the database level -- The `CLNSIG` field confirms the classification - -**What to expect:** -- 0-10 pathogenic/likely pathogenic hits is typical for a 30X WGS -- Most pathogenic hits are **carrier status** (heterozygous) for recessive conditions — you carry one copy but aren't affected -- A heterozygous pathogenic variant in a recessive gene (like GJB2 for hearing loss) means you're a **carrier**, not affected -- A homozygous pathogenic variant, or a heterozygous variant in a dominant gene, needs attention - -**When to worry:** -- Pathogenic variant in a **dominant** gene (one copy is enough to cause disease) -- **Two** pathogenic variants in the same recessive gene (one from each parent) -- Any variant in cancer predisposition genes (BRCA1, BRCA2, MLH1, MSH2, etc.) - -### 2. PharmCAT Report (Step 7) - -**What it tells you:** How your genes affect drug metabolism. These results are clinically relevant and should be shared with your prescribing physician. - -**Where to look:** `${SAMPLE}/pharmcat/` (Nextflow) or `${SAMPLE}/vcf/` (bash scripts) — PharmCAT writes its reports there. Open the HTML report in a browser. - -**Key genes to check:** - -| Gene | Affects | Common Impact | -|---|---|---| -| CYP2C19 | PPIs, clopidogrel, SSRIs, voriconazole | Rapid metabolizers burn through drugs too fast | -| CYP2D6 | Codeine, tramadol, tamoxifen, many psych meds | Poor metabolizers get toxic buildup | -| CYP2C9 | Warfarin, NSAIDs, phenytoin | Dose adjustment needed | -| DPYD | 5-fluorouracil (cancer drug) | Poor metabolizers can die from standard doses | -| SLCO1B1 | Statins (simvastatin, atorvastatin) | Increased myopathy risk | -| NAT2 | Isoniazid (TB), hydralazine | Slow acetylators have more side effects | -| UGT1A1 | Irinotecan, atazanavir | Two \*28 alleles are associated with Gilbert syndrome (elevated bilirubin) | - -**What to do:** Share the PharmCAT report with your prescribing physician or pharmacist. PharmCAT is a research tool — its authors explicitly note that missing positions, unphased input, and undetected structural variation (especially CYP2D6) can affect genotype and phenotype calls. The report is a valuable starting point for pharmacogenomic-guided prescribing, but clinical confirmation may be warranted before making medication changes, especially for high-risk drugs (DPYD, CYP2D6-dependent opioids). - -### 3. CPSR Report (Step 17) - -**What it tells you:** Cancer predisposition screening using CPSR's curated cancer gene panels (panel 0 covers 500+ genes). Step 17 also turns on CPSR's secondary findings: the ACMG SF list, which includes cardiac and metabolic genes outside cancer (see [Before you look](#before-you-look-what-the-pipeline-can-tell-you)). - -**Where to look:** `${SAMPLE}/cpsr/` — open the HTML report in a browser. - -**How to read it:** -- CPSR puts every variant in its genes into one of the five ACMG/AMP classes: **Pathogenic**, **Likely pathogenic**, **VUS** (uncertain significance), **Likely benign** and **Benign**. A variant that already has a ClinVar entry can carry ClinVar's class instead of the one CPSR computed. -- Only **Pathogenic** and **Likely pathogenic** call for clinical attention. The report lists them first, and the secondary findings in their own section. - ---- - -## Structural Variants (Steps 4, 5, 15, 18, 19, 22) - -### What Are Structural Variants? - -Unlike SNPs (single letter changes), structural variants are large rearrangements: -- **Deletions (DEL):** A chunk of DNA is missing -- **Duplications (DUP):** A chunk is copied extra times -- **Inversions (INV):** A chunk is flipped backwards -- **Translocations (BND):** A chunk moved to a different chromosome -- **Insertions (INS):** New DNA inserted - -### A Note About BND (Breakend) Calls - -If you run Manta or Delly, you will see many **BND** calls — often hundreds or thousands. BND indicates a "breakend" where one end of a read pair maps to a different chromosome or a distant location. This sounds alarming ("translocation!") but: - -- **Most BND calls are artifacts** of repetitive regions, segmental duplications, or mobile elements -- A typical genome has 1,000-3,000 BND calls from Manta and 5,000+ from Delly -- **Fewer than 5 are likely real** inter-chromosomal translocations in a healthy genome -- BND calls require **multi-caller support** (called by both Manta and Delly at overlapping breakpoints) to be considered credible -- Unless a BND disrupts a known disease gene AND is confirmed by a second caller, it can be safely ignored - -### How Many Is Normal? - -A typical human genome has: -- ~5,000-10,000 structural variants total -- Most are in non-coding regions and harmless -- ~5-20 may affect genes -- 0-2 may be clinically significant - -### Which Callers to Trust? - -If you ran multiple SV callers: -- **Called by 2+ callers (Manta + Delly, or Manta + CNVpytor):** Lower false-positive rate -- **Called by 1 caller only:** Lower confidence, may be false positive -- **duphold DHFFC < 0.7 for deletions:** High confidence (depth drops as expected) -- **duphold DHBFC > 1.3 for duplications:** High confidence (depth rises as expected) - -### SV Consensus (Step 22) - -`${SAMPLE}/sv_merged/${SAMPLE}_sv_consensus.vcf.gz` keeps the SVs that two or more callers found (Manta, Delly, CNVpytor and, when they ran, Sniffles2 and TIDDIT): SURVIVOR pairs two calls when both breakpoints lie within 1 kb of each other and their type and strands agree. `SUPP` says how many callers found each one and `SUPP_VEC` which. GRIDSS is left out (its breakend records never match). It is the short list to read first, but it drops real SVs that only one caller found; see [step 22](22-survivor-merge.md#limitations). - -### AnnotSV Output - -The AnnotSV TSV (step 5) adds clinical annotations to each SV: -- `ACMG_class`: 1 (benign) to 5 (pathogenic) -- `Overlapped_CDS_percent`: How much of a gene is affected -- Focus on SVs with `ACMG_class` 4 or 5 that overlap known disease genes - ---- - -## STR Expansions (Step 9) - -### What Are Repeat Expansions? - -Some regions of DNA have short sequences repeated many times (e.g., CAG CAG CAG...). When the number of repeats exceeds a threshold, it can cause disease. - -### How to Read ExpansionHunter Output - -The output VCF lists each tested locus with the number of repeats found. Key loci: - -| Locus | Gene | Normal | Intermediate / Premutation | Pathogenic | Disease | -|---|---|---|---|---|---| -| HTT | HTT | <=26 | 27-35 (mutable normal); 36-39 (reduced penetrance) | >=40 (full penetrance) | Huntington's disease | -| FMR1 | FMR1 | <45 | 45-54 (intermediate); 55-200 (premutation) | >200 | Fragile X syndrome | -| ATXN1 | ATXN1 | <33 | — | >39 | Spinocerebellar ataxia 1 | -| C9orf72 | C9orf72 | <24 | 24-30 (gray zone, lab cutoffs vary) | Typically hundreds-thousands | ALS / Frontotemporal dementia | -| DMPK | DMPK | <35 | 35-49 (premutation) | >=50 | Myotonic dystrophy type 1 | - -**HTT 36-39 repeats (reduced penetrance):** Individuals in this range may or may not develop Huntington's disease. The risk increases with repeat length but is not certain. GeneReviews classifies >=40 as full penetrance and 36-39 as reduced penetrance. Alleles of 27-35 ("mutable normal") do not cause disease but may expand in offspring. - -**C9orf72 gray zone:** The exact pathogenic threshold for C9orf72 is not established. Laboratory cutoffs are discordant (JNNP 2021 review). Clearly pathogenic expansions are typically hundreds to thousands of repeats. Short-read WGS has limited ability to size very large expansions accurately. - -**FMR1 intermediate zone (45-54 repeats):** Not affected, but repeats may expand in offspring. Carriers should receive genetic counseling. Premutation (55-200) carries risk of FXTAS (males >50) and FXPOI. - -The report prints no all-clear verdict for these loci. It lists every locus in the ExpansionHunter output, the five above first, with its repeat count, Stranger's status and the catalog's normal maximum (`STR_NORMAL_MAX`), and, when step 9b (Stranger) ran, every locus Stranger marks `pre_mutation` or `full_mutation`; without step 9b, compare the counts with the table yourself. It also says: - -- **Not sized.** A locus whose call rests on flanking reads only prints "size not determined (flanking reads only)"; one with in-repeat reads only (longer than a read) prints its count as at least the lower end of ExpansionHunter's interval; one with fewer than 2 spanning reads, or an interval that reaches the catalog's pathogenic minimum, is marked low confidence. Such a locus is listed apart and never shown as normal, because an unsized repeat can hide an expansion. -- **Intermediate range.** A sized count inside the HTT, FMR1 or DMPK intermediate range of the table above prints the range, a note that it does not cause disease but may expand in offspring, and the GeneReviews chapter. The report reads these ranges from `bin/str_intermediate_ranges.tsv`, and `tests/test_str_ranges.py` checks that the table above prints the same ones. -- **RFC1.** A `full_mutation` flag at RFC1 carries the motif verdict of step 9b ([RFC1 note](09b-stranger.md#notes)). - -A count under every threshold is not a guarantee: repeat-expansion calls from short reads can be wrong at some loci, and a long expansion cannot be sized. Intermediate-range results should be discussed with a genetic counselor. - -### Stranger Annotation (Step 9b) - -Step 9b adds a `STR_STATUS` field to each locus in the VCF so you do not need to look up thresholds manually: - -| STR_STATUS | Meaning | -|---|---| -| `normal` | Repeat count is within the established normal range | -| `pre_mutation` | Elevated repeat count; not currently disease-causing but carries risk of expansion in offspring or late-onset carrier effects (e.g. FXTAS, FXPOI for FMR1) | -| `full_mutation` | Repeat count exceeds the pathogenic threshold for this locus | - -The `Disease`, `InheritanceMode`, `SourceDisplay`, `STR_NORMAL_MAX`, and `STR_PATHOLOGIC_MIN` INFO fields give the clinical context for each locus (a field the catalog has no value for prints as `.`). Use a VCF viewer or `bcftools query` to extract them: - -```bash -bcftools query -f '%INFO/STR_STATUS\t%INFO/Disease\t%INFO/InheritanceMode\t%INFO/SourceDisplay\t%INFO/STR_NORMAL_MAX\t%INFO/STR_PATHOLOGIC_MIN\n' \ - expansion_hunter/_eh_stranger.vcf -``` - -`full_mutation` at any locus warrants follow-up with a clinical geneticist. Short-read WGS has limited sizing accuracy for very large expansions (>150 repeats), so confirmatory testing may be recommended. - ---- - -## Telomere Length (Step 10) - -### What It Means - -Telomeres are protective caps at the ends of chromosomes that shorten with cell division. TelomereHunter measures `tel_content`: intratelomeric reads per million reads with 48-52% GC. It is a relative telomere content, not a telomere length, and the report labels it "Telomere content (relative)". - -### How to Interpret - -- **Higher `tel_content` = more telomeric reads** (generally correlates with longer telomeres) -- **Lower `tel_content` = fewer telomeric reads** (generally correlates with shorter telomeres) -- There is no universal "normal" range — compare between samples of similar age, sequenced on the same platform -- Typical `tel_content` for 30X WGS: 300-800 (varies by sequencing platform and coverage) - -### Limitations - -- **Not a clinical telomere length measurement.** TelomereHunter was developed for cancer genome analysis, not as a validated healthy-population aging assay -- **Not a "biological age" readout.** While telomere length correlates with aging at a population level, it is a rough estimate of aging rate and is not established as a clinically important standalone risk marker for individuals (see Vaiserman & Krasnienkov, "Telomere Length as a Marker of Biological Age," 2021) -- Short-read WGS systematically underestimates telomere length compared to dedicated assays (TRF, FlowFISH) -- Useful only for **relative comparisons** between samples run on the same platform — not absolute measurements and not individual health predictions +1. [ClinVar Screen (Step 6)](results/clinvar.md#clinvar-screen-step-6){ #1-clinvar-screen-step-6 } +2. [PharmCAT Report (Step 7)](results/pharmacogenomics.md#pharmcat-report-step-7){ #2-pharmcat-report-step-7 } +3. [CPSR Report (Step 17)](results/secondary-findings.md#cpsr-report-step-17){ #3-cpsr-report-step-17 } --- -## ROH Analysis (Step 11) - -### What It Means - -Runs of Homozygosity (ROH) are long stretches where both copies of your DNA are identical. Everyone has some ROH, but extensive ROH can indicate: -- Parental relatedness (consanguinity) -- Uniparental disomy -- Population bottleneck effects - -### How to Read - -Add up the autosomal segments of 5 Mb or more and compare the total with the table in [step 11](11-roh-analysis.md#total-roh-and-parental-relationship), which gives the expected total for each parental relationship (about 45 Mb for second cousins, 180 Mb for first cousins). In short: - -- **No segments of 5 Mb or more, or only a few:** no sign that your parents are related -- **Many small segments (1-5 Mb):** population-level background, typical of population isolates (Ashkenazi, Finnish, etc.) -- **One chromosome carrying most of the long runs:** possible uniparental disomy rather than related parents. The reports flag a chromosome when its segments of 5 Mb or more add up to at least 20 Mb and to more than half of all such autosomal segments ([step 11](11-roh-analysis.md#one-chromosome-carrying-most-of-the-runs-possible-uniparental-disomy)) - -### Centromeric Artifacts - -Some apparent ROH near centromeres are artifacts of low-coverage sequencing in repetitive regions. These can be ignored. - ---- - -## Mitochondrial Haplogroup (Step 12) - -### What It Means - -Your mitochondrial haplogroup traces your maternal ancestry lineage. It's determined by the specific set of variants in your mitochondrial DNA (inherited only from your mother). - -### Common European Haplogroups - -| Haplogroup | Origin | Notes | -|---|---|---| -| H | Western Europe | Most common in Europe (~40%) | -| U | Northern/Eastern Europe | Second most common (~15%) | -| T | Near East / Mediterranean | ~10% of Europeans | -| K | Near East | ~6% of Europeans, Ashkenazi ~30% | -| J | Near East | ~8% of Europeans | -| V | Iberian Peninsula / Scandinavia | ~4% | -| I | Near East / Europe | ~3% | - -### Medical Relevance - -Mitochondrial haplogroups have weak associations with some diseases (Parkinson's, diabetes, longevity), but these are population-level statistics, not individual predictions. The main clinical value is in step 20 (GATK Mutect2 mitochondrial mode), which detects disease-causing mitochondrial variants and heteroplasmy. - -### Exploring Your Branch - -Both reports link your haplogroup to FamilyTreeDNA Discover (free, no upload) and to YFull's tree. Discover shows the branch's age, where its testers live and the ancient DNA samples on it. For the Y haplogroup of step 37 the reports add the same two links. The free and paid routes, what each one takes and what an upload shares: [step 12](12-mito-haplogroup.md#explore-your-branch-further) and [step 37](37-y-haplogroup.md#explore-your-branch-further). The pipeline never uploads anything. - ---- - -## VEP Annotation (Step 13) - -### What It Adds - -VEP annotates every variant with: -- **Consequence type:** missense, nonsense, synonymous, splice site, etc. -- **SIFT score:** Predicts if amino acid change is tolerated (>0.05) or damaging (<0.05) -- **PolyPhen score:** Predicts if change is benign (<0.15), possibly damaging (0.15-0.85), or probably damaging (>0.85) -- **gnomAD frequencies:** How common this variant is in gnomAD's exomes (`gnomADe_AF`) and genomes (`gnomADg_AF`) - -### gnomAD Frequency: Your Best Sanity Check - -The single most useful annotation VEP adds is the **gnomAD allele frequency** — how common a variant is in the general population. Step 13 runs VEP with `--everything`, which turns on both `--af_gnomade` and `--af_gnomadg`, so each variant gets two frequencies: `gnomADe_AF` from gnomAD's exomes and `gnomADg_AF` from its genomes. A non-coding variant outside exome capture regions has no exome frequency but can still have a genome frequency, so check both before calling a variant absent from gnomAD. The recipes below do that. - -**Key principle:** A variant that is common in healthy people is almost certainly benign, regardless of what any prediction tool says. - -| gnomAD AF | Interpretation | Action | -|---|---|---| -| > 5% (0.05) | Common polymorphism | Benign. Ignore. | -| 1-5% | Low-frequency variant | Almost certainly benign | -| 0.1-1% | Uncommon | Probably benign, but check ClinVar | -| 0.01-0.1% | Rare | Worth investigating if in a disease gene | -| < 0.01% | Very rare | Potentially significant. Check ClinVar + literature | -| Absent | Novel or ultra-rare | Could be significant OR a sequencing artifact. Verify with a second method | - -**If a variant is "pathogenic" in ClinVar but has gnomAD AF > 1%:** The ClinVar entry may be outdated or wrong. Truly pathogenic variants for severe diseases are almost always rare (< 0.1%) because natural selection removes them from the population. - -### Filtering Strategy - -Step 23 (the [clinical filter](#clinical-filter-step-23)) already applies the filters most people want. For your own queries, use the [Quick Variant Filtering Recipes](#quick-variant-filtering-recipes) below. Do not `grep` the VEP VCF: VEP writes its values by position inside the pipe-separated `CSQ` field, never as `name=value`, so a grep for `gnomAD_AF` or `SYMBOL=` matches nothing, and a grep for `HIGH` or `1/1` can match text in other fields. `bcftools +split-vep` reads the fields by name. - -### What "HIGH Impact" Means - -VEP classifies variant impact as: - -| Impact | Types | Interpretation | -|---|---|---| -| HIGH | Stop gained, frameshift, splice donor/acceptor | Likely breaks the protein | -| MODERATE | Missense, in-frame insertion/deletion | Changes the protein, may or may not matter | -| LOW | Synonymous, splice region | Probably no functional effect | -| MODIFIER | Intronic, intergenic, UTR | Usually non-functional | - -**Everyone has ~100-150 HIGH impact variants.** Most are in one copy (heterozygous) of non-essential genes. Don't panic at the number. - ---- - -## Pathogenicity Scores (Step 30 — vcfanno) - -If you ran step 30, your VCF now includes quantitative pathogenicity scores beyond VEP's qualitative SIFT/PolyPhen predictions. These scores are widely used as computational evidence in variant interpretation (see ClinGen's PP3/BP4 calibration framework). - -### CADD (Combined Annotation Dependent Depletion) - -Scores **all** variant types (coding, non-coding, splice, regulatory). Uses a PHRED-like scale where higher = more deleterious. - -| CADD PHRED | Interpretation | Context | -|---|---|---| -| < 10 | Likely benign | Bottom 90% of genome variation | -| 10-20 | Uncertain | Top 10%, but most are still benign | -| 20-25 | Potentially deleterious | Top 1% — investigate if in a disease gene | -| 25-30 | Likely deleterious | Top 0.3% — strong candidate for pathogenicity | -| > 30 | Highly deleterious | Top 0.1% — likely damaging if in a constrained gene | - -**When to use CADD:** Best for non-coding and splice-region variants where SIFT/PolyPhen don't apply. For missense variants, REVEL and AlphaMissense are more specific. - -### REVEL (Rare Exome Variant Ensemble Learner) - -Scores **missense variants only**. Combines 13 individual tools into a single 0-1 score. Recommended by ClinGen for ACMG PP3/BP4 evidence. - -The levels below are the REVEL row of Table 2 in [Pejaver et al. 2022](https://doi.org/10.1016/j.ajhg.2022.10.013), the ClinGen calibration of PP3/BP4: - -| REVEL Score | ClinGen Evidence Level | Interpretation | -|---|---|---| -| <= 0.003 | BP4_Very Strong | Very strong evidence of benign | -| > 0.003 to <= 0.016 | BP4_Strong | Strong evidence of benign | -| > 0.016 to <= 0.183 | BP4_Moderate | Moderate evidence of benign | -| > 0.183 to <= 0.290 | BP4_Supporting | Supporting evidence of benign | -| > 0.290 to < 0.644 | No evidence | Uncertain significance | -| >= 0.644 to < 0.773 | PP3_Supporting | Supporting evidence of pathogenicity | -| >= 0.773 to < 0.932 | PP3_Moderate | Moderate evidence of pathogenicity | -| >= 0.932 | PP3_Strong | Strong evidence of pathogenicity | - -There is no PP3 Very Strong level for REVEL: no score reached it in the calibration. - -**When to use REVEL:** First-line score for evaluating missense variants. If REVEL >= 0.644, investigate the variant seriously. - -### AlphaMissense - -DeepMind's protein-structure-informed **missense** classifier. Uses AlphaFold2 protein structure predictions to assess amino acid substitution impact. - -| am_pathogenicity | am_class | Interpretation | -|---|---|---| -| < 0.34 | likely_benign | Predicted benign by protein structure analysis | -| 0.34-0.564 | ambiguous | Uncertain — use other evidence | -| > 0.564 | likely_pathogenic | Predicted damaging based on protein structure | - -These cut-offs are AlphaMissense's own class boundaries, set by its authors. They are not ACMG/ClinGen evidence levels. - -**When to use AlphaMissense:** Complements REVEL as a second opinion. If they disagree, investigate further. Agreement does not add up to stronger PP3 evidence: ClinGen's calibration recommends one tool, used genome-wide, for PP3/BP4, so picking whichever tool scores a variant highest would bias the result. - -### SpliceAI - -Deep learning model predicting **splice-altering** variants. Scores four types of splice disruption: acceptor gain (AG), acceptor loss (AL), donor gain (DG), donor loss (DL). - -| Max Delta Score | Interpretation | -|---|---| -| < 0.2 | Unlikely to affect splicing | -| 0.2-0.5 | May affect splicing — investigate | -| 0.5-0.8 | Likely affects splicing | -| > 0.8 | Strong evidence of splice disruption | - -**When to use SpliceAI:** VEP already flags canonical splice site variants (GT/AG dinucleotides). SpliceAI catches **cryptic** splice variants — intronic or exonic variants that create new splice sites or disrupt existing ones through more subtle mechanisms. - -### gnomAD Gene Constraint (Step 23 summary) - -These are per-gene metrics (not per-variant) added to the clinical filter summary TSV. They measure how intolerant a gene is to different types of mutations. - -| Metric | Threshold | Meaning | -|---|---|---| -| LOEUF < 0.35 | Constrained for loss-of-function | Gene is intolerant to LoF mutations — a HIGH impact variant here is more likely to cause disease | -| pLI >= 0.9 | Loss-of-function intolerant | Same as LOEUF but older metric. LOEUF is preferred. | -| mis_Z > 3.09 | Constrained for missense | Gene is intolerant to missense mutations — a REVEL-high missense here is more concerning | - -**Combining scores:** A rare variant (gnomAD AF < 0.01%) with CADD > 25, in a constrained gene (LOEUF < 0.35), with ClinVar pathogenic classification, is a high-confidence pathogenic finding. Any one of these alone is insufficient. - ---- - -### Quick Variant Filtering Recipes - -Copy-paste these commands to extract the most clinically relevant variants. They read step 13's output, `${GENOME_DIR}/${SAMPLE}/vep/${SAMPLE}_vep.vcf.gz`, with `bcftools +split-vep` from the pipeline's pinned bcftools image, which picks each value out of the `CSQ` field by name (the same way step 23 does). `-s worst` keeps the most severe consequence of each variant; `-d` keeps every transcript. - -```bash -# Run from the repository root, with GENOME_DIR and SAMPLE set -source versions.env -bcf() { docker run --rm -i -v "${GENOME_DIR}:/genome" -w /genome "${BCFTOOLS_IMAGE}" bcftools "$@"; } -VEP_VCF="${SAMPLE}/vep/${SAMPLE}_vep.vcf.gz" # relative to GENOME_DIR - -# Rare: below 0.1% in gnomAD exomes and genomes, or absent from them -RARE='(gnomADe_AF="." || gnomADe_AF<0.001) && (gnomADg_AF="." || gnomADg_AF<0.001)' -FIELDS='%CHROM\t%POS\t%REF\t%ALT\t%SYMBOL\t%Consequence\t%gnomADe_AF\t%gnomADg_AF\t[%GT]\n' - -# 1. Homozygous high-impact variants (stop gained, frameshift, splice donor/acceptor) -bcf view -i 'GT="AA"' "$VEP_VCF" | - bcf +split-vep - -s worst -i 'IMPACT="HIGH"' -f "$FIELDS" - -# 2. Rare high-impact variants -bcf +split-vep "$VEP_VCF" -s worst -i "IMPACT=\"HIGH\" && ${RARE}" -f "$FIELDS" - -# 3. Compound heterozygous candidates: genes with two or more rare heterozygous -# HIGH or MODERATE variants. The phase is unknown: both can sit on the same -# copy of the gene, so this is a list to curate, not a finding. -bcf view -i 'GT="het"' "$VEP_VCF" | - bcf +split-vep - -s worst -i "(IMPACT=\"HIGH\" || IMPACT=\"MODERATE\") && ${RARE}" -f '%SYMBOL\n' | - sort | uniq -c | awk '$1 >= 2' | sort -rn - -# 4. HIGH or MODERATE variants in genes you name, matched on the exact symbol. -# The list here is a few cancer genes for illustration; CPSR (step 17) already -# reports the full ACMG secondary-findings list. For pharmacogenes, use -# GENES="CYP2D6 CYP2C19 CYP2C9 DPYD UGT1A1 SLCO1B1 TPMT NUDT15". -GENES="BRCA1 BRCA2 MLH1 MSH2 MSH6 PMS2 APC MUTYH TP53" -bcf +split-vep "$VEP_VCF" -d -i 'IMPACT="HIGH" || IMPACT="MODERATE"' -f "$FIELDS" | - awk -F'\t' -v genes="$GENES" 'BEGIN { n = split(genes, g, " "); for (i = 1; i <= n; i++) want[g[i]] = 1 } $5 in want' | - sort -u - -# 5. Rare missense variants that SIFT and PolyPhen both call damaging -bcf +split-vep "$VEP_VCF" -s worst \ - -i "Consequence~\"missense_variant\" && SIFT~\"^deleterious[(]\" && PolyPhen~\"^probably_damaging\" && ${RARE}" \ - -f "$FIELDS" - -# 6. Missense variants absent from both gnomAD sets (novel or ultra-rare) -bcf +split-vep "$VEP_VCF" -s worst \ - -i 'Consequence~"missense_variant" && gnomADe_AF="." && gnomADg_AF="."' -f "$FIELDS" -``` - -If the VCF came from a VEP run without gnomAD frequencies, `+split-vep` stops with `the tag "gnomADe_AF" is not defined`: drop the `RARE` term and the two gnomAD columns from `FIELDS`. PharmCAT (step 7) misses some alleles, and CYP2D6 needs the BAM-based callers (steps 21 and 32), so recipe 4 with the pharmacogene list is a cross-check, not a replacement. - -**Important:** These are starting points, not definitive screens. Any interesting finding should be cross-referenced with ClinVar and ideally confirmed by a second method (Sanger sequencing or a clinical lab). - ---- - -## CNVpytor Results (Step 18) - -CNVpytor detects **copy number variants** using read depth analysis — complementary to Manta's paired-end/split-read approach. - -**Where to look:** `${SAMPLE}/cnvpytor/${SAMPLE}_cnvs.txt` - -**Format:** Each line has: type, region, size, normalized_RD, e-value1, e-value2, e-value3, e-value4, q0 - -**What to expect:** -- 3,000-4,000 total CNVs (mostly deletions) -- 1,500-2,000 significant (e-value < 0.01) -- Calls at chromosome starts (chr1:1-10000) are telomeric artifacts — ignore them - -**Filtering:** -```bash -# Significant CNVs only (e-value < 0.01) -awk '$5 < 0.01' ${SAMPLE}_cnvs.txt - -# Large deletions (>100kb, potentially clinically relevant) -awk '$1 == "deletion" && $3 > 100000 && $5 < 0.01' ${SAMPLE}_cnvs.txt - -# Large duplications -awk '$1 == "duplication" && $3 > 100000 && $5 < 0.01' ${SAMPLE}_cnvs.txt -``` - -**Multi-caller overlap:** CNVs found by both Manta AND CNVpytor have lower false-positive rates. Cross-reference by checking if the same genomic region appears in both output files. - ---- - -## Delly Results (Step 19) - -Delly is a third structural variant caller, detecting deletions, duplications, inversions, and translocations. - -**Where to look:** `${SAMPLE}/delly/${SAMPLE}_sv.vcf.gz` - -**Quick summary:** -```bash -# Count SVs by type -bcftools query -f '%INFO/SVTYPE\n' ${SAMPLE}_sv.vcf.gz | sort | uniq -c - -# Filter PASS variants only -bcftools view -f PASS ${SAMPLE}_sv.vcf.gz | grep -cv '^#' -``` - -**What to expect:** -- 5,000-15,000 total SV calls -- Most are small deletions (<1kb) -- PASS filter reduces count significantly - -**Multi-caller overlap:** SVs detected by Manta + Delly + CNVpytor (or any 2 of 3) have substantially lower false-positive rates. Single-caller calls, especially large ones, should be viewed with caution. - ---- - -## Mitochondrial Variants (Step 20) - -GATK Mutect2 in mitochondrial mode detects variants with heteroplasmy fractions — the proportion of your mitochondria carrying each variant. - -**Where to look:** `${SAMPLE}/mito/${SAMPLE}_chrM_filtered.vcf.gz` - -**Key field:** `AF` (allele fraction) indicates heteroplasmy level: - -| AF Level | Meaning | -|---|---| -| >0.95 | Homoplasmic — fixed in all mitochondria (haplogroup-defining) | -| 0.10-0.95 | Heteroplasmic — clinically significant range | -| 0.03-0.10 | Low-level heteroplasmy — often age-related somatic | -| <0.03 | Near detection limit | - -**What to expect:** -- 50-70 PASS variants total -- 25-35 homoplasmic (haplogroup variants) -- 25-35 low-level heteroplasmic (mostly <5%) -- Length variants in the D-loop poly-C tracts (m.303-315 and m.16184-16193) in most samples: the reports list them apart and do not count them as heteroplasmy - -**When to investigate further:** -- Heteroplasmic variant at a known disease position: the report lists each heteroplasmic call, and each call ClinVar classes as anything but benign, with its ClinVar class and gnomAD v3.1 frequency (`mito/_chrM_annotated.tsv`). [MITOMAP](https://www.mitomap.org/) has the curated status; look it up by hand, as it refuses scripted requests -- m.3243A>G (MELAS) or m.8344A>G (MERRF) at detectable heteroplasmy levels -- Any position in MT-ATP6, MT-ND genes with AF >0.10 - -**Important caveats about heteroplasmy thresholds:** -- There is **no single absolute heteroplasmy threshold** that determines clinical significance. Thresholds vary by variant and by tissue (ClinGen/MSeqDR mtDNA interpretation specifications) -- **Blood and saliva underrepresent heteroplasmy** for many mitochondrial diseases. WGS from saliva, a cheek swab or blood may show lower heteroplasmy levels than affected tissues (muscle, nerve). m.3243A>G in particular shows different clinical phenotypes at very different heteroplasmy levels across tissues -- The AF values from this pipeline reflect the tissue your sample came from. A low or absent heteroplasmy level there does **not** rule out clinically significant heteroplasmy in other tissues -- For any detected pathogenic mtDNA variant, discuss with a specialist who can order tissue-specific testing if warranted - -**Cross-reference:** Compare with step 12 (haplogrep3) — your homoplasmic variants should match your assigned haplogroup. - ---- - -## Somatic Variants (Step 29) [EXPERIMENTAL] - -Mutect2 in tumor-only mode looks for somatic mutations -- variants acquired during your lifetime rather than inherited. Consumer WGS is usually made from saliva or a cheek swab, a mix of cheek cells and white blood cells; the main category of interest is **clonal hematopoiesis (CHIP)**, which lives in the blood cells. - -**Where to look:** `${SAMPLE}/somatic/${SAMPLE}_somatic_filtered.vcf.gz` - -**Key field:** `AF` (allele fraction) indicates the clone size: - -| AF Range | Likely Source | -|---|---| -| 0.45-0.55 | Heterozygous germline (false positive) | -| ~1.0 | Homozygous germline (false positive) | -| 0.10-0.40 | Could be a large somatic clone, mosaic, or noisy germline | -| below 0.10 | At 30X this is one to three reads: mostly noise, see below | - -**What to expect:** -- Thousands of PASS calls in a healthy individual -- the vast majority are germline false positives -- At 30X a variant in 2% of the reads has less than one supporting read on average, and one in 10% about three. Only variants at roughly 10% allele fraction or more can be told apart from noise. That is an approximate rule about reads, not a clone size: a heterozygous variant at 10% allele fraction sits in about 20% of the sampled cells, and copy number and the mix of cell types in the sample shift it. Most CHIP (defined from 2% allele fraction) is invisible here, and a clean result does not rule it out -- Without a matched normal sample, germline variants that are rare in gnomAD will often pass all filters - -**CHIP genes to check:** DNMT3A, TET2, ASXL1, TP53, JAK2, SF3B1, SRSF2, PPM1D, CBL. CHIP prevalence increases with age and is associated with elevated cardiovascular risk and risk of hematologic malignancies. - -**Important caveats:** -- This step has a much higher false positive rate than any other step in the pipeline -- Do not interpret PASS variants as confirmed somatic without cross-referencing with the germline VCF (step 3) and gnomAD frequencies (step 13) -- If a variant is also called at ~50% AF by DeepVariant in step 3, it is almost certainly germline - ---- - -## Reads, Alignment, Variant Calling and Coverage (Steps 1b, 2, 3, 16, 16b) - -These steps check that the data is good enough for everything else. Look at them first if a later result looks strange. - -- **fastp (step 1b):** `${SAMPLE}/fastq_trimmed/${SAMPLE}_fastp.html`. A few percent of reads trimmed or dropped is normal; a large share points at a library or upload problem. -- **Alignment and variant calling (steps 2 and 3):** the [example output](#variant-calling-step-3) below shows what a 30X genome looks like: about 4.5 to 5.5 million variants and a Ti/Tv ratio of 2.0 to 2.1. -- **Coverage (step 16b, mosdepth):** `${SAMPLE}/mosdepth/${SAMPLE}.mosdepth.summary.txt`. The `total` row should show a mean near the depth you paid for (about 30). Below 15, small-variant calls lose accuracy. See [step 16b](16b-mosdepth.md#interpreting-results). -- **Sex check (step 16, indexcov):** `${SAMPLE}/indexcov/` estimates the copy number of chrX and chrY from the BAM index. About 1 and 1 is XY, about 2 and 0 is XX. A result that does not match the sex you gave the pipeline means a sample swap, a mislabelled file or a real sex-chromosome difference; [step 16](16-indexcov.md#interpretation) lists the patterns. Re-check the input before reading anything else. - -## HLA Typing (Step 8) - -**Where to look:** `${SAMPLE}/hla_t1k/${SAMPLE}_hla_genotype.tsv`, two alleles per HLA gene. - -The main use is drug safety: a few HLA alleles predict severe reactions to specific drugs, for example HLA-B\*57:01 with abacavir and HLA-B\*58:01 with allopurinol. The report marks each allele of `bin/hla_alleles.tsv` (the drug alleles, and disease-associated ones such as HLA-B\*27 and HLA-C\*06:02) present or absent, and derives the coeliac DQ types (DQ2.5, DQ2.2, DQ8) and the number of DQB1\*02 copies from the DQA1 and DQB1 rows; [step 8](08-hla-typing.md#what-the-report-derives) gives the rules and the sources. The genotypes are unphased, so DQ2.5 reads "likely cis" or "possible trans". These lines are information about alleles, not a diagnosis: carrying one does not mean the condition or reaction will happen, and not carrying one does not rule it out. A quality-0 gene with many tied candidates reads "no confident call", and the HLA-HFE row is a sequence allele, not the HFE C282Y/H63D genotype, which comes from the variant calls: the ClinVar hits list each change, and the HFE rule of the traits step reads the two together ([step 42](42-traits.md)). Typing from short-read WGS is approximate. HLA-A and HLA-B also reach PharmCAT (step 36), so the PharmCAT report and the CPIC recommendations (step 27) give the drug guidance for them; the CPIC file marks them `[outside call]`. If one of those alleles appears, or is missing and you are about to take the drug, ask for a clinical HLA test; do not rely on this output for transplant matching. With `HLA_QC=true` (opt-in) the HLA section also says, for HLA-A, HLA-B and HLA-C, whether OptiType agrees with T1K at two fields, with both calls; a gene they disagree on, or did not compare, is not passed to PharmCAT when it is HLA-A or HLA-B. With `KIR=true` (opt-in) the KIR genes are typed too, in `${SAMPLE}/kir_t1k/`, and the HLA section lists the genes T1K typed; no step interprets them. - -## Clinical Filter (Step 23) - -**Where to look:** `${SAMPLE}/clinical/${SAMPLE}_clinical_summary.tsv`, one row per variant with gene, impact, scores and gnomAD constraint. - -Step 23 keeps the HIGH-impact variants, the MODERATE ones below 1% in gnomAD exomes, the ClinVar pathogenic ones and, when step 30 ran, the variants that CADD, SpliceAI, REVEL or AlphaMissense flag. Expect a few hundred rows. Most are heterozygous variants in genes that tolerate one broken copy. Read the rows in constrained genes (low LOEUF, see [gene constraint](#gnomad-gene-constraint-step-23-summary)) first, then the homozygous ones. The HIGH-impact rows have no frequency filter, so check gnomAD for each before you worry. [Step 23](23-clinical-filter.md#what-gets-filtered) gives the exact rules. - -## Variant Prioritization (Step 31) - -**Where to look:** `${SAMPLE}/slivar/${SAMPLE}_slivar_summary.tsv`, `${SAMPLE}/slivar/${SAMPLE}_compound_hets.tsv` and `${SAMPLE}/slivar/${SAMPLE}_compound_het_pairs.tsv`. - -slivar sorts the damaging variants into four groups (rare HIGH, rare MODERATE with damaging scores, ClinVar pathogenic, and frequency unknown: HIGH or MODERATE with no population frequency, so not called rare) and lists genes where you carry two such variants. The genotypes alone cannot tell whether the two variants sit on different copies of the gene (in trans, which can cause recessive disease) or on the same copy (in cis, which usually does not), so step 31 reads the phase from the reads: `${SAMPLE}_compound_het_pairs.tsv` says `trans`, `cis` or `unknown` for each pair, with the reads that say so. Only variants close enough for one read pair to cover both (a few hundred bases) can be phased this way, so most pairs read `unknown`. Expect a thousand or more candidate pairs; nearly all are noise. A `trans` pair matters only in a gene that fits your health history, and an `unknown` one needs a parent's DNA or long reads to phase. See [step 31](31-slivar.md#interpretation). - -## More Pharmacogenomics: Cyrius, CPIC, pypgx and the Consensus (Steps 21, 27, 32, 36) - -- **CPIC lookup (step 27):** `${SAMPLE}/cpic/${SAMPLE}_cpic_recommendations.txt` turns PharmCAT's calls into the drugs with CPIC guidance. Only genes whose phenotype names a changed function or a positive test get drug entries. Genes PharmCAT called with no such phenotype (`n/a`, the reference VKORC1 genotype `-1639 GG`, `Uncertain Susceptibility`) are listed in their own section without drugs; PharmCAT's HTML report has any guidance for them. Genes PharmCAT could not call are listed separately at the end; their absence from the drug list does not mean normal function. Its section "Calls From Other Tools" says which of HLA-A, HLA-B and CYP2D6 reached PharmCAT from the BAM-based callers, and why the others did not. -- **PGx consensus (step 36):** `${SAMPLE}/pgx_consensus/${SAMPLE}_pgx_consensus.tsv` shows, for HLA-A, HLA-B and CYP2D6, what each caller said and what was passed to PharmCAT. CYP2D6 is passed only when pypgx and Cyrius give the same diplotype and the depth at CYP2D6 passed its check; otherwise it reads `indeterminate`. -- **pypgx (step 32):** `${SAMPLE}/pypgx/${SAMPLE}_pypgx_summary.tsv` calls the 29 genes of `assets/pgx_genes.tsv`, 13 of them (the `bam` rows) with copy number from the BAM (GSTT1, which GRCh38 has only on the ALT contig `chr22_KI270879v1_alt`, is reported as not callable on the default no-ALT reference; a BAM aligned to a reference with ALT or HLA contigs stops the step with a `REFUSED` line, because its depth splits between a gene and its ALT copy), so it sees gene deletions and duplications PharmCAT cannot. `${SAMPLE}_pharmcat_comparison.tsv`, written into the same folder by step 27 (CPIC lookup), shows where the two tools agree. -- **Cyrius (step 21, opt-in, non-commercial licence):** `${SAMPLE}/cyrius/${SAMPLE}_cyp2d6.tsv` gives a second CYP2D6 call from the BAM. Without it, CYP2D6 stays `indeterminate`. - -CYP2D6 is the hard gene: a nearby pseudogene and frequent copy-number changes confuse short reads. The pipeline gives CYP2D6 drug guidance only when two callers agree; even then, take any result that would change a prescription to a pharmacist or a certified pharmacogenomics test first. - -## SMN1 and SMN2 Copy Number (Step 35, opt-in) - -**Where to look:** `${SAMPLE}/paralogs/${SAMPLE}_smn_copy_number.tsv`, one row per stretch of the SMN1/SMN2 locus with the copy number of the two genes together (agCN) and of each (psCN), each with a quality. - -One SMN1 copy suggests SMA carrier status; two do not rule it out (two copies on one chromosome, none on the other). Trust a value only with quality 20 or more and filter `PASS`, and confirm anything that matters with a clinical SMN1 test. [Step 35](35-paralogs.md#interpreting-results) explains the columns. - -**The silent-carrier marker:** `${SAMPLE}/paralogs/${SAMPLE}_smn1_marker.tsv` reads SMN1 g.27134T>G (rs143838139), and both reports and the live person page print it right under the SMN1 copies: `present`, `uncertain`, `not seen`, `too little depth` or `not assessed`. With two SMN1 copies, `present` raises the chance that both sit on one chromosome (2+0, a silent carrier) and `not seen` lowers it; it proves neither, and the figures depend on ancestry. A carrier test in a clinical laboratory confirms it. [The SMN1 silent-carrier marker](35-paralogs.md#the-smn1-silent-carrier-marker) gives the thresholds. - -## Lp(a) and the LPA KIV-2 Count (Step 46, opt-in) - -**Where to look:** `${SAMPLE}/lpa/${SAMPLE}_lpa_kiv2.tsv`, the KIV-2 repeat count over both chromosomes with its 95% interval and class, and `${SAMPLE}/lpa/${SAMPLE}_lpa_kiv2_4925.tsv`, the KIV-2 4925G>A splice variant. Both reports put the count beside the two LPA risk variants of step 42 (rs10455872, rs3798220), and print the 4925G>A line right under the count: `present` or `uncertain` with the reads that carry it and the copies, `not seen`, `too little depth` or `not assessed`. - -Fewer repeats or a risk allele point to a higher Lp(a); a 4925G>A copy lowers the Lp(a) made from the chromosome that carries it. Neither tells which chromosome carries what, and `not seen` is not proof of absence at low depth. The count is a research estimate: the Lp(a) blood test is the measure, once in every adult. [Step 46](46-lpa.md) explains the columns. - -## Mobile Element Insertions (Step 50, opt-in) - -**Where to look:** `${SAMPLE}/mei/${SAMPLE}_mei_sites.tsv` for the three known disease insertions (SVA insertions in FKTN, TAF1 and BBS1), and `${SAMPLE}/mei/${SAMPLE}_mei.tsv` for every Alu, L1 and SVA insertion MEGAnE called, with one or two copies. - -Expect many calls: a genome carries about 1,200 such insertions, nearly all common and harmless. Read the catalog first. **Seen** means an insertion of the same element was called inside the site's window; one copy of the FKTN or BBS1 insertion is carrier status for a recessive condition, and the TAF1 insertion on a male's single X is what causes X-linked dystonia-parkinsonism. **Not seen** is not proof of absence: short reads find an SVA's edges, not its body. **Not callable** means too few reads there (under 10x) to tell. Confirm anything that matters in a clinical laboratory. [Step 50](50-mei.md) explains the columns. - -## Ancient Populations (Step 44, opt-in) - -**Where to look:** `${SAMPLE}/ancient/${SAMPLE}_ancient.tsv`, the 20 ancient groups nearest to you on a PCA of present-day West Eurasians, with their dates, places and number of people (`n`), and `${SAMPLE}/ancient/${SAMPLE}_ancient_pca.svg`, the plot with you marked. - -Closeness on the PCA is genetic similarity, not descent: being near an ancient group does not mean you descend from those people. A group with `n` of 1 or 2 has a noisy position. The axes describe West Eurasia only, so a genome with most of its ancestry from elsewhere lands at the edge and its nearest groups say little. `sites_used` in `${SAMPLE}_ancient_info.tsv` tells how many panel sites your gVCF covered; a whole genome covers nearly all of them (1,133,951 of the 1,150,106 that lift to GRCh38, for a gVCF built from the public GIAB HG002 genotypes). [Step 44](44-ancient.md) explains the columns. - -## Polygenic Risk Scores (Step 25) - -**Where to look:** `${SAMPLE}/prs/${SAMPLE}_prs_summary.tsv`, one row per condition, and the PRS section of the report. - -With the ancestry panel installed (`scripts/setup.sh --ancestry-panel`), each score has a **Percentile** and an **Ancestry_Group**: the 80th percentile means your score is higher than 80% of the reference samples whose genetic ancestry is most similar to yours (step 26 names that group). A percentile is still not a probability: most people at a high percentile never get the condition, and lifestyle and family history often weigh more. Percentiles of different conditions are not comparable either. - -Without the panel the report says "Raw score only": the sum cannot be compared with anyone, so read nothing into its size. A score with `NA` as its sum matched under 75% of its variants and was not scored; `Matched_Pct` says how many. When `Input` is `vcf` (no gVCF), the sums leave out the sites where you match the reference ([step 25](25-prs.md#interpreting-results) explains why). When `ChrX` says `left out`, the sample's sex was not given and the score's chrX rows are not in its sum ([chrX and the sex](25-prs.md#chrx-and-the-sex)). The breast cancer score is for women and the prostate cancer score for men: when you declared the other sex, `Sex_Check` says `not applicable to the declared sex` and the score has no percentile. The blood pressure and lipid scores rank a genetic tendency only; a measured blood pressure or lipid panel says far more. The Alzheimer's score includes APOE: read [Before you look](#before-you-look-what-the-pipeline-can-tell-you) first. - -## Reports (Steps 24 and 28) - -- **HTML report (step 24):** `${SAMPLE}/${SAMPLE}_report.html` collects the headline results of the other steps on one page, including the ClinVar hits table. It contains health findings: share it only as you would share a medical record. -- **MultiQC (step 28):** `${SAMPLE}/multiqc/multiqc_report.html` puts the QC of fastp, samtools, mosdepth and the other tools in one page. It is about data quality, not about your health. +## The Report Chapters + +Each chapter of the report has its own page in this guide. The links below keep the old addresses of the sections on this page working. + +- [Secondary Findings and Carrier Status](results/secondary-findings.md) + - [Carrier Status Is Not Disease](results/secondary-findings.md#carrier-status-is-not-disease){ #carrier-status-is-not-disease } + - [Clinical Filter (Step 23)](results/secondary-findings.md#clinical-filter-step-23){ #clinical-filter-step-23 } + - [Variant Prioritization (Step 31)](results/secondary-findings.md#variant-prioritization-step-31){ #variant-prioritization-step-31 } +- [ClinVar and Variant Annotation](results/clinvar.md) + - [The VUS Trap](results/clinvar.md#the-vus-trap){ #the-vus-trap } + - [ClinVar Star Ratings](results/clinvar.md#clinvar-star-ratings){ #clinvar-star-ratings } + - [VEP Annotation (Step 13)](results/clinvar.md#vep-annotation-step-13){ #vep-annotation-step-13 }: [What It Adds](results/clinvar.md#what-it-adds){ #what-it-adds }, [gnomAD Frequency: Your Best Sanity Check](results/clinvar.md#gnomad-frequency-your-best-sanity-check){ #gnomad-frequency-your-best-sanity-check }, [Filtering Strategy](results/clinvar.md#filtering-strategy){ #filtering-strategy }, [What "HIGH Impact" Means](results/clinvar.md#what-high-impact-means){ #what-high-impact-means } + - [Pathogenicity Scores (Step 30 — vcfanno)](results/clinvar.md#pathogenicity-scores-step-30-vcfanno){ #pathogenicity-scores-step-30-vcfanno }: [CADD](results/clinvar.md#cadd-combined-annotation-dependent-depletion){ #cadd-combined-annotation-dependent-depletion }, [REVEL](results/clinvar.md#revel-rare-exome-variant-ensemble-learner){ #revel-rare-exome-variant-ensemble-learner }, [AlphaMissense](results/clinvar.md#alphamissense){ #alphamissense }, [SpliceAI](results/clinvar.md#spliceai){ #spliceai }, [gnomAD Gene Constraint (Step 23 summary)](results/clinvar.md#gnomad-gene-constraint-step-23-summary){ #gnomad-gene-constraint-step-23-summary } + - [Quick Variant Filtering Recipes](results/clinvar.md#quick-variant-filtering-recipes){ #quick-variant-filtering-recipes } +- [Pharmacogenomics and HLA](results/pharmacogenomics.md) + - [HLA Typing (Step 8)](results/pharmacogenomics.md#hla-typing-step-8){ #hla-typing-step-8 } + - [More Pharmacogenomics: Cyrius, CPIC, pypgx and the Consensus (Steps 21, 27, 32, 36)](results/pharmacogenomics.md#more-pharmacogenomics-cyrius-cpic-pypgx-and-the-consensus-steps-21-27-32-36){ #more-pharmacogenomics-cyrius-cpic-pypgx-and-the-consensus-steps-21-27-32-36 } +- [Structural Variants, Copy Number and Repeats](results/structural-and-repeats.md) + - [Structural Variants (Steps 4, 5, 15, 18, 19, 22)](results/structural-and-repeats.md#structural-variants-steps-4-5-15-18-19-22){ #structural-variants-steps-4-5-15-18-19-22 }: [What Are Structural Variants?](results/structural-and-repeats.md#what-are-structural-variants){ #what-are-structural-variants }, [A Note About BND (Breakend) Calls](results/structural-and-repeats.md#a-note-about-bnd-breakend-calls){ #a-note-about-bnd-breakend-calls }, [How Many Is Normal?](results/structural-and-repeats.md#how-many-is-normal){ #how-many-is-normal }, [Which Callers to Trust?](results/structural-and-repeats.md#which-callers-to-trust){ #which-callers-to-trust }, [SV Consensus (Step 22)](results/structural-and-repeats.md#sv-consensus-step-22){ #sv-consensus-step-22 }, [AnnotSV Output](results/structural-and-repeats.md#annotsv-output){ #annotsv-output } + - [STR Expansions (Step 9)](results/structural-and-repeats.md#str-expansions-step-9){ #str-expansions-step-9 }: [What Are Repeat Expansions?](results/structural-and-repeats.md#what-are-repeat-expansions){ #what-are-repeat-expansions }, [How to Read ExpansionHunter Output](results/structural-and-repeats.md#how-to-read-expansionhunter-output){ #how-to-read-expansionhunter-output }, [Stranger Annotation (Step 9b)](results/structural-and-repeats.md#stranger-annotation-step-9b){ #stranger-annotation-step-9b } + - [CNVpytor Results (Step 18)](results/structural-and-repeats.md#cnvpytor-results-step-18){ #cnvpytor-results-step-18 } + - [Delly Results (Step 19)](results/structural-and-repeats.md#delly-results-step-19){ #delly-results-step-19 } + - [SMN1 and SMN2 Copy Number (Step 35, opt-in)](results/structural-and-repeats.md#smn1-and-smn2-copy-number-step-35-opt-in){ #smn1-and-smn2-copy-number-step-35-opt-in } + - [Lp(a) and the LPA KIV-2 Count (Step 46, opt-in)](results/structural-and-repeats.md#lpa-and-the-lpa-kiv-2-count-step-46-opt-in){ #lpa-and-the-lpa-kiv-2-count-step-46-opt-in } + - [Mobile Element Insertions (Step 50, opt-in)](results/structural-and-repeats.md#mobile-element-insertions-step-50-opt-in){ #mobile-element-insertions-step-50-opt-in } +- [Ancestry, Lineage and Polygenic Scores](results/ancestry-and-lineage.md) + - [ROH Analysis (Step 11)](results/ancestry-and-lineage.md#roh-analysis-step-11){ #roh-analysis-step-11 }: [What It Means](results/ancestry-and-lineage.md#what-it-means){ #what-it-means_1 }, [How to Read](results/ancestry-and-lineage.md#how-to-read){ #how-to-read }, [Centromeric Artifacts](results/ancestry-and-lineage.md#centromeric-artifacts){ #centromeric-artifacts } + - [Mitochondrial Haplogroup (Step 12)](results/ancestry-and-lineage.md#mitochondrial-haplogroup-step-12){ #mitochondrial-haplogroup-step-12 }: [What It Means](results/ancestry-and-lineage.md#what-it-means_1){ #what-it-means_2 }, [Common European Haplogroups](results/ancestry-and-lineage.md#common-european-haplogroups){ #common-european-haplogroups }, [Medical Relevance](results/ancestry-and-lineage.md#medical-relevance){ #medical-relevance }, [Exploring Your Branch](results/ancestry-and-lineage.md#exploring-your-branch){ #exploring-your-branch } + - [Mitochondrial Variants (Step 20)](results/ancestry-and-lineage.md#mitochondrial-variants-step-20){ #mitochondrial-variants-step-20 } + - [Ancient Populations (Step 44, opt-in)](results/ancestry-and-lineage.md#ancient-populations-step-44-opt-in){ #ancient-populations-step-44-opt-in } + - [Polygenic Risk Scores (Step 25)](results/ancestry-and-lineage.md#polygenic-risk-scores-step-25){ #polygenic-risk-scores-step-25 } +- [Telomeres and Somatic Variants](results/traits-and-somatic.md) + - [Telomere Length (Step 10)](results/traits-and-somatic.md#telomere-length-step-10){ #telomere-length-step-10 }: [What It Means](results/traits-and-somatic.md#what-it-means){ #what-it-means }, [How to Interpret](results/traits-and-somatic.md#how-to-interpret){ #how-to-interpret }, [Limitations](results/traits-and-somatic.md#limitations){ #limitations } + - [Somatic Variants (Step 29) [EXPERIMENTAL]](results/traits-and-somatic.md#somatic-variants-step-29-experimental){ #somatic-variants-step-29-experimental } +- [Data Quality and Reports](results/quality.md) + - [Reads, Alignment, Variant Calling and Coverage (Steps 1b, 2, 3, 16, 16b)](results/quality.md#reads-alignment-variant-calling-and-coverage-steps-1b-2-3-16-16b){ #reads-alignment-variant-calling-and-coverage-steps-1b-2-3-16-16b } + - [Reports (Steps 24 and 28)](results/quality.md#reports-steps-24-and-28){ #reports-steps-24-and-28 } --- @@ -819,7 +203,7 @@ NOTE: Centromeric ROH (chr1:125-143MB, chr9:42-60MB, chr18:15-20MB) are technica No line under the heading means no segment of 5 Mb or more: no evidence of related parents. If there are some, add them up and compare the total with the table in [step 11](11-roh-analysis.md#total-roh-and-parental-relationship). -### Telomere Length (Step 10) +### Telomere Length (Step 10) { #telomere-length-step-10_1 } ``` tel_content: diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index 71c5e7d8a..038d182a3 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -534,7 +534,7 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted ### PharmCAT reads MT-RNR1 only as an outside call - **Failed:** MT-RNR1 (aminoglycoside-induced hearing loss) was No Result on every sample, although step 20 calls chrM with Mutect2 at high depth. - **Why:** PharmCAT 3.4.0 has no MT-RNR1 allele definition for its matcher, so chrM records in its VCF input are never matched for the gene. It lists MT-RNR1 among the genes it prefers as an outside call, and its own MT-RNR1 test passes the gene that way. -- **Fix:** `og pgx mito-merge` (was `bin/pgx_mito_merge.py`) turns step 20's chrM calls into one MT-RNR1 outside call (a homoplasmic named allele, or `Reference` only when the BAM covers every named position), added to step 36's outside calls. Step 27 checks only CYP2D6, HLA-A and HLA-B against step 36's table: MT-RNR1 never appears there, and checking it marked the new call stale. The details are in [Pharmacogenomics, Input the preprocessor or PharmCAT refuses](07-pharmacogenomics.md#input-the-preprocessor-or-pharmcat-refuses). +- **Fix:** `og pgx mito-merge` (was `bin/pgx_mito_merge.py`) turns step 20's chrM calls into one MT-RNR1 outside call (a named allele at AF 0.05 or more, heteroplasmic included, or `Reference` only when the BAM covers every named position), added to step 36's outside calls. Step 27 checks only CYP2D6, HLA-A and HLA-B against step 36's table: MT-RNR1 never appears there, and checking it marked the new call stale. The details are in [Pharmacogenomics, Input the preprocessor or PharmCAT refuses](07-pharmacogenomics.md#input-the-preprocessor-or-pharmcat-refuses). ### PharmCAT's log file needs a writable working directory - **Failed:** every step 07 run printed a logback status dump with a `pharmcat.log (Permission denied)` stack trace. The calls were not affected. diff --git a/docs/live-site.md b/docs/live-site.md index b2b16746b..d49ac0161 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,27 +56,49 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a copy-number dot (a CHIP call is a finding of its own card, so the CHIP card closes whole), a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. APOE is closed on the genome map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. A child's page has the report gate below. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. +**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person any family lists with a father or a mother, whatever the other families that list them say. While a family folder cannot be read, a person it might list is treated as a child too, so the gate fails closed: everyone while its members cannot be read (a members file that names no one counts as one that cannot be read), and the people it lists while another of its files cannot be read (then `family.json` answers 500 with a fixed line). The server works this out from the files on each request, so once the files read again the people open again, with no other request needed. A person whose state cannot be read is treated as a child, and a child whose results cannot be read stays closed too. The server gives the answer to the person page in `state.json` (the family plugin's `child`, or `unknown` for a person no readable family lists), and the home page, the family page and the finding addresses decide by the same rule. That answer alone decides whether a page keeps results closed: an adult's page is their normal page even while `family.json` has not answered or fails. While the gate is shut, the page shows only: + +- a fixed explanation and the number of results to act on ("1 result to act on", "2 results to act on"); +- one "Show the result" button per closed result, all in one block; +- for each result opened on purpose, its own record (title, detail, explanation, plan, evidence and caveat) with a button to close it again, in the order they were opened. + +Nothing else of the child's report reaches the page: no cards, tables, lead, counts by level, genome map, charts, images, file links, search box or progress view. The title is fixed. The page shows the same thing whichever result is closed. + +Which result a button opens is drawn at random on each page load, so neither a button's place nor what it opens follows the report's order. The buttons have opaque names in the address, `#f/closed-1` and `#f/opened-1`. Any other `#f/` and any query land on `#f/closed-1`. On a finding address of a gated child the server answers the same page for every well-formed id, and the page then shows the run's own address and the same closed page, so a guessed address cannot tell whether the child has that result. + +Opening the last result loads the page again, and the full report shows, as an adult's does, with its own addresses (`#f/`). A map or section address returns to the default view. On a finding address, the finding's page opens when the results have that finding; any other id opens the run's page, as the server answers 404 for it once the report is open. The open report keeps results closed until opened, as before the gate, so its search says "Results this page keeps closed are left out of the search until they are opened." under the hits. Only which results were opened crosses that load: a one-use ticket in session storage with a digest of the results' records, bound to that one load by a random value in the window's name. The page reads and removes it as it starts, before any network read, so a slow server does not lose the opening. Closing a result again (the genome map's "Close it again") clears the page at once and loads it again closed, without its search, caches or views; that result is closed, and the others stay opened. A result to act on that appears or changes while the report is open closes it the same way. A reload, or a page shown again with Back, closes every result again. + +The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway, with the same options, and hands each view its first answer for that file, whatever its status. So each file is fetched as often, in the same order and with the same options as without the gate. For an adult the gate reads only the state; the views then ask for the family and the results themselves. For a child the gate also reads the results before any view gets an answer. In live mode the person page asks for the family and for its results at once, on its first state, so for a child the gate asks for both, in that order, as soon as the state has answered. The two answers race as they do without the gate, and they reach the views in the order they arrived: the gate checks a child's results on the text it already read, so no answer overtakes another. The marks a section address sets (the contents' `aria-current`, the card's highlight) therefore come out as without the gate: they stay when the family answers first, and the redraw that the family's answer starts clears them when the results answer first. The event stream opens when `live.js` asks for it, right after `state.json`; its messages wait until the gate knows whether the page is a child's, then reach the page in order. One thing differs for a child: its views get their first answer once its results have answered, so its page draws later by the time the results take. + +One count is left out for a child whose report is closed. Other members' pages, a parent's for example, list the family's members with how many cards each report flags. That count changes with which cards the child's unopened results flag, so `family.json` gives `attention: null` for that child. While the child is gated on the family page, the child's member line reads "Report closed" and the pedigree has no count, never "Nothing flagged". Once every result to act on of the child is opened there, the page shows the count `family.json` gives without the gate, that of the child's `summary.json`, which the child's `results.json` carries as `saved_attention`. For adults and children with no result to act on, the count in `family.json` and the member line are as before, "Nothing flagged" for a count that cannot be read included. Each member run in `family.json` also has `child`: whether that person's pages treat them as a child, by the rule above. So a person any family lists as a child is closed on every family page that lists them, a partner's family too. There they keep the role that family gives them: they are not counted as its child, and a couple they are part of keeps its line, with its screen closed. + +The home page decides from `/runs.json` alone. A gated child is a child (the rule above, read from the families of `/runs.json`) whose entry counts or names a result to act on. The server counts a child's results from the summary its `results.json` reads. While `summary.json` is older than a finished step, that is the step files read again, so the home page closes a child whenever the child's page does. A child with no result to act on in either summary keeps the numbers of `summary.json`, as before. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. + +On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. A gated person whom this family lists as a parent or partner, because another family lists them as a child, gets their buttons under the members, after the member list. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of (the couple keeps its names and whether it was screened; a screened couple's line says its screen stays closed). The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. + +The gate holds the page's `fetch` and `EventSource`. A view or plugin reads a run's files through them. `XMLHttpRequest`, an ``, an `